@1aboveio/app-dev-skills 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +29 -2
  2. package/package.json +33 -4
  3. package/skills/airflow/LICENSE +3 -0
  4. package/skills/airflow/SKILL.md +199 -0
  5. package/skills/airflow/release-check.mjs +93 -0
  6. package/skills/android-foundation/LICENSE +3 -0
  7. package/skills/android-foundation/SKILL.md +192 -0
  8. package/skills/android-foundation/assets/android-ci-job.yml +217 -0
  9. package/skills/android-foundation/references/ci.md +87 -0
  10. package/skills/android-foundation/references/gotchas.md +221 -0
  11. package/skills/android-foundation/references/setup.md +164 -0
  12. package/skills/android-foundation/release-check.mjs +93 -0
  13. package/skills/android-foundation/scripts/device-run.sh +176 -0
  14. package/skills/android-foundation/scripts/device-run.test.mjs +172 -0
  15. package/skills/app-debug/LICENSE +3 -0
  16. package/skills/app-debug/SKILL.md +110 -0
  17. package/skills/app-debug/references/common-errors.md +128 -0
  18. package/skills/app-debug/release-check.mjs +93 -0
  19. package/skills/better-auth/LICENSE +3 -0
  20. package/skills/better-auth/SKILL.md +598 -0
  21. package/skills/better-auth/references/feishu-api.md +270 -0
  22. package/skills/better-auth/release-check.mjs +93 -0
  23. package/skills/ios-foundation/LICENSE +3 -0
  24. package/skills/ios-foundation/SKILL.md +184 -0
  25. package/skills/ios-foundation/references/setup.md +243 -0
  26. package/skills/ios-foundation/release-check.mjs +93 -0
  27. package/skills/monorepo/LICENSE +3 -0
  28. package/skills/monorepo/SKILL.md +462 -0
  29. package/skills/monorepo/release-check.mjs +93 -0
  30. package/skills/nextjs-fullstack/LICENSE +3 -0
  31. package/skills/nextjs-fullstack/SKILL.md +211 -0
  32. package/skills/nextjs-fullstack/conventions.md +318 -0
  33. package/skills/nextjs-fullstack/frontend-conventions.md +61 -0
  34. package/skills/nextjs-fullstack/nextjs16.md +287 -0
  35. package/skills/nextjs-fullstack/release-check.mjs +93 -0
  36. package/skills/nextjs-fullstack/server-actions.md +409 -0
  37. package/skills/prisma-setup/LICENSE +3 -0
  38. package/skills/prisma-setup/SKILL.md +181 -0
  39. package/skills/prisma-setup/nextjs.md +258 -0
  40. package/skills/prisma-setup/release-check.mjs +93 -0
  41. package/skills/prisma-setup/turborepo.md +301 -0
  42. package/skills/pyspark/LICENSE +3 -0
  43. package/skills/pyspark/SKILL.md +127 -0
  44. package/skills/pyspark/assets/templates/etl.py +748 -0
  45. package/skills/pyspark/assets/templates/utils/__init__.py +1 -0
  46. package/skills/pyspark/assets/templates/utils/hudi_metadata.py +14 -0
  47. package/skills/pyspark/references/diagnosis-and-profiling.md +169 -0
  48. package/skills/pyspark/references/etl-contract.md +129 -0
  49. package/skills/pyspark/references/production-validation.md +131 -0
  50. package/skills/pyspark/references/reconciliation.md +38 -0
  51. package/skills/pyspark/references/transformation-design.md +190 -0
  52. package/skills/pyspark/references/velocity-feature-calculation.md +193 -0
  53. package/skills/pyspark/release-check.mjs +93 -0
  54. package/skills/pyspark/scripts/spark_eventlog_summary.py +223 -0
  55. package/skills/python-backend/LICENSE +3 -0
  56. package/skills/python-backend/SKILL.md +325 -0
  57. package/skills/python-backend/assets/uv-install.sh +48 -0
  58. package/skills/python-backend/references/uv-installs.md +83 -0
  59. package/skills/python-backend/release-check.mjs +93 -0
  60. package/skills/shadcn/LICENSE +3 -0
  61. package/skills/shadcn/SKILL.md +120 -0
  62. package/skills/shadcn/assets/shadcn-small.png +0 -0
  63. package/skills/shadcn/assets/shadcn.png +0 -0
  64. package/skills/shadcn/cli.md +411 -0
  65. package/skills/shadcn/customization.md +224 -0
  66. package/skills/shadcn/mcp.md +101 -0
  67. package/skills/shadcn/release-check.mjs +93 -0
  68. package/skills/shadcn/rules/base-vs-radix.md +323 -0
  69. package/skills/shadcn/rules/component-selection.md +67 -0
  70. package/skills/shadcn/rules/composition.md +195 -0
  71. package/skills/shadcn/rules/data-table.md +201 -0
  72. package/skills/shadcn/rules/forms.md +255 -0
  73. package/skills/shadcn/rules/icons.md +103 -0
  74. package/skills/shadcn/rules/styling.md +167 -0
  75. package/skills/zod-v4/LICENSE +3 -0
  76. package/skills/zod-v4/SKILL.md +286 -0
  77. package/skills/zod-v4/release-check.mjs +93 -0
@@ -0,0 +1,748 @@
1
+ from abc import ABC, abstractmethod
2
+ from argparse import ArgumentParser
3
+ from contextlib import contextmanager
4
+ from dataclasses import dataclass
5
+ import enum
6
+ import logging
7
+ import os
8
+ import sys
9
+ import threading
10
+ from typing import List, Dict, Union, Literal, Optional
11
+
12
+ from dateutil.relativedelta import relativedelta
13
+ import pandas as pd
14
+ from pyspark.sql import SparkSession, DataFrame
15
+ from pyspark.sql.functions import *
16
+ from pyspark.sql.types import *
17
+ from pyspark.sql.utils import AnalysisException
18
+ import typer
19
+ from typing_extensions import Annotated
20
+
21
+ from utils.hudi_metadata import hudi_metadata_write_options
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class HudiLayout:
28
+ """Target on-disk layout. Do not pair with Spark/Hudi shuffle parallelism."""
29
+
30
+ files_per_partition: int = 1
31
+ target_file_size_mb: int = 128
32
+
33
+ def write_options(self) -> Dict[str, Union[str, int]]:
34
+ target_bytes = int(self.target_file_size_mb) * 1024 * 1024
35
+ group_bytes = target_bytes * max(1, int(self.files_per_partition))
36
+ return {
37
+ "hoodie.parquet.max.file.size": str(target_bytes),
38
+ "hoodie.parquet.small.file.limit": str(target_bytes),
39
+ "hoodie.copyonwrite.insert.split.size": str(target_bytes),
40
+ "hoodie.clustering.plan.strategy.target.file.max.bytes": str(target_bytes),
41
+ "hoodie.clustering.plan.strategy.max.bytes.per.group": str(group_bytes),
42
+ }
43
+
44
+
45
+ _SPARK_LEASE_LOCK = threading.RLock()
46
+ _SPARK_LEASES = {}
47
+
48
+
49
+ def _classic_spark_runtime_exists() -> bool:
50
+ from pyspark import SparkContext
51
+
52
+ session = (
53
+ SparkSession.getActiveSession()
54
+ or getattr(SparkSession, "_instantiatedSession", None)
55
+ )
56
+ if (
57
+ session is not None
58
+ and getattr(getattr(session, "_sc", None), "_jsc", None) is not None
59
+ ):
60
+ return True
61
+ context = getattr(SparkContext, "_active_spark_context", None)
62
+ return context is not None and getattr(context, "_jsc", None) is not None
63
+
64
+
65
+ def _connect_spark_runtime_exists() -> bool:
66
+ module = sys.modules.get("pyspark.sql.connect.session")
67
+ connect_session = getattr(module, "SparkSession", None) if module else None
68
+ if connect_session is None:
69
+ return False
70
+ return (
71
+ connect_session.getActiveSession() is not None
72
+ or getattr(connect_session, "_default_session", None) is not None
73
+ )
74
+
75
+
76
+ def _spark_runtime_exists(spark_remote: Optional[str]) -> bool:
77
+ connect_mode = bool(
78
+ spark_remote
79
+ or os.environ.get("SPARK_REMOTE")
80
+ or os.environ.get("SPARK_CONNECT_MODE_ENABLED")
81
+ )
82
+ return (
83
+ _connect_spark_runtime_exists()
84
+ if connect_mode
85
+ else _classic_spark_runtime_exists()
86
+ )
87
+
88
+
89
+ def create_spark_session(spark_remote: Optional[str] = None) -> SparkSession:
90
+ """Create a Spark session in Spark Connect or classic spark-submit mode."""
91
+ builder = SparkSession.builder
92
+ if spark_remote:
93
+ builder = builder.remote(spark_remote)
94
+ return builder.getOrCreate()
95
+
96
+
97
+ @contextmanager
98
+ def managed_spark_session(
99
+ spark_remote: Optional[str] = None,
100
+ *,
101
+ builder=None,
102
+ ):
103
+ """Lease a Spark session and stop only a runtime created by this helper."""
104
+ with _SPARK_LEASE_LOCK:
105
+ runtime_existed = _spark_runtime_exists(spark_remote)
106
+ spark = (
107
+ builder.getOrCreate()
108
+ if builder is not None
109
+ else create_spark_session(spark_remote)
110
+ )
111
+ key = id(spark)
112
+ state = _SPARK_LEASES.get(key)
113
+ if state is None:
114
+ state = {
115
+ "leases": 0,
116
+ "helper_owned": not runtime_existed,
117
+ }
118
+ _SPARK_LEASES[key] = state
119
+ state["leases"] += 1
120
+
121
+ try:
122
+ yield spark
123
+ finally:
124
+ with _SPARK_LEASE_LOCK:
125
+ state = _SPARK_LEASES.get(key)
126
+ if state is None:
127
+ return
128
+ state["leases"] -= 1
129
+ if state["leases"] == 0:
130
+ del _SPARK_LEASES[key]
131
+ if state["helper_owned"]:
132
+ spark.stop()
133
+
134
+
135
+ class Etl(ABC):
136
+ """
137
+ The base class for ETL jobs.
138
+
139
+ Attributes:
140
+ ----------
141
+ id: str
142
+ field name of the unique key, required in Hudi table
143
+ ts: str
144
+ field name of the record unique timestamp, used as Hudi `preCombine` field
145
+ filter_by: str
146
+ field name filtering in `extract` function
147
+
148
+ Methods:
149
+ -------
150
+ extract(start_date: str, end_date: str):
151
+ extract table from <`src_db`.`src_tbl`>
152
+
153
+ transform(df: DataFrame):
154
+ performs transformation to df
155
+
156
+ load(df):
157
+ load into <`dst_db`.`dst_tbl`> as `spark_table` or `hudi_table` based on the `table_type` setting
158
+ """
159
+
160
+ table_type: Literal["hudi_table", "spark_table"] = "hudi_table"
161
+ hudi_mode: Literal["upsert", "insert_overwrite", "insert_overwrite_table"] = (
162
+ "upsert"
163
+ )
164
+ src_db = None
165
+ src_tbl = None
166
+ dst_db = None
167
+ dst_tbl = None
168
+ id = None
169
+ ts = None
170
+ filter_by = None
171
+ offset = None
172
+ par_cols = []
173
+ path = None
174
+ repartition = []
175
+ url: str = None
176
+ concurrency_mode: Literal["SINGLE_WRITER", "OPTIMISTIC_CONCURRENCY_CONTROL"] = (
177
+ "SINGLE_WRITER"
178
+ )
179
+ extra_hudi_options: Dict[str, Union[str, bool, int]] = {}
180
+ hudi_layout = HudiLayout(
181
+ files_per_partition=1,
182
+ target_file_size_mb=128,
183
+ )
184
+ zookeeper: str = None
185
+
186
+ def __init__(
187
+ self,
188
+ start_date: str = None,
189
+ end_date: str = None,
190
+ zookeeper: str = None,
191
+ url: str = None,
192
+ bulk: bool = True,
193
+ hudi_mode_override: str = None,
194
+ **kwargs,
195
+ ):
196
+ fields = ["dst_db", "dst_tbl", "path"]
197
+ if self.table_type == "hudi_table":
198
+ fields += ["id", "ts"]
199
+ for field in fields:
200
+ if getattr(self, field) is None:
201
+ raise ValueError(
202
+ f"Field {field} cannot be None. Please set {field} before proceed"
203
+ )
204
+ if len(self.par_cols) == 0:
205
+ logger.warning("`par_cols` is empty; output will not be partitioned")
206
+ if type(self.repartition) != list:
207
+ self.repartition = [self.repartition]
208
+ if self.filter_by is None:
209
+ logger.warning(
210
+ "`filter_by` is not set, will extract full table, please be aware!"
211
+ )
212
+ self.start_date: str = start_date if start_date else None
213
+ self.end_date: str = end_date if end_date else None
214
+ self.zookeeper = zookeeper if zookeeper else self.zookeeper
215
+ self.url = url if url else self.url
216
+ self.bulk = bulk
217
+ # Allow CLI override of hudi_mode (e.g. --hudi-mode bulk_insert)
218
+ if hudi_mode_override:
219
+ self.hudi_mode = hudi_mode_override
220
+ for k, v in kwargs.items():
221
+ setattr(self, k, v)
222
+
223
+ if (
224
+ self.concurrency_mode == "OPTIMISTIC_CONCURRENCY_CONTROL"
225
+ and self.zookeeper is None
226
+ ):
227
+ raise ValueError("concurrency control enabled but zookeeper not set")
228
+ self.spark = SparkSession.getActiveSession()
229
+
230
+ def __call__(self):
231
+ self.process()
232
+
233
+ @classmethod
234
+ def run_from_cli(
235
+ cls,
236
+ start_date: Annotated[str, typer.Option("--start-date")] = None,
237
+ end_date: Annotated[str, typer.Option("--end-date")] = None,
238
+ zookeeper: Annotated[str, typer.Option("--zookeeper")] = None,
239
+ url: Annotated[str, typer.Option("--url")] = None,
240
+ bulk: Annotated[bool, typer.Option("--bulk/--per-day")] = True,
241
+ ):
242
+ obj = cls.__new__(cls)
243
+ obj.__init__(start_date, end_date, zookeeper, url, bulk)
244
+ obj()
245
+ return obj
246
+
247
+ def extract(self, start_date: str = None, end_date: str = None):
248
+ start_date = start_date or self.start_date
249
+ end_date = end_date or self.end_date
250
+ df = self.spark.table(f"{self.src_db}.{self.src_tbl}")
251
+
252
+ if self.offset and start_date:
253
+ from dateutil.relativedelta import relativedelta
254
+ from dateutil.parser import parse as date_parse
255
+
256
+ start_date = date_parse(start_date) - relativedelta(days=self.offset)
257
+ start_date = start_date.strftime("%Y-%m-%d")
258
+
259
+ if self.filter_by and start_date:
260
+ df = df.filter(col(self.filter_by) >= start_date)
261
+ if self.filter_by and end_date:
262
+ df = df.filter(col(self.filter_by) < end_date)
263
+ return df
264
+
265
+ def select_or_none(
266
+ self, df: DataFrame, cols: Union[List, Dict[str, Union[str, DataType]]]
267
+ ) -> DataFrame:
268
+ if type(cols) == list:
269
+ available_cols = [i for i in cols if i in df.columns]
270
+ missing_cols = [i for i in cols if i not in df.columns]
271
+ df = df.select(*available_cols)
272
+
273
+ logger.warning(
274
+ f"Columns {missing_cols} are missing, filling with None and cast as default StringType"
275
+ )
276
+ for col in missing_cols:
277
+ df = df.withColumn(col, lit(None).cast(StringType()))
278
+ elif type(cols) == dict:
279
+ available_cols = [i for i in cols.keys() if i in df.columns]
280
+ missing_cols = [i for i in cols.keys() if i not in df.columns]
281
+ df = df.select(*available_cols)
282
+ for col in missing_cols:
283
+ col_type = cols[col]
284
+ df = df.withColumn(col, lit(None).cast(col_type))
285
+ else:
286
+ raise TypeError(f"cols {cols} should be List or Dict")
287
+ df = df.select(*cols)
288
+ return df
289
+
290
+ @abstractmethod
291
+ def transform(self, df: DataFrame) -> DataFrame:
292
+ return df
293
+
294
+ def _prepare_partition(self, df: DataFrame) -> DataFrame:
295
+ self.spark.conf.set("spark.sql.sources.partitionOverwriteMode", "DYNAMIC")
296
+ new_cols = [i for i in df.columns if i not in self.par_cols] + self.par_cols
297
+ df = df.select(*new_cols)
298
+ if self.start_date and self.filter_by:
299
+ df = df.filter(col(self.filter_by) >= self.start_date)
300
+ return df
301
+
302
+ def _is_existing_hudi_table(self) -> bool:
303
+ """Check if a Hudi table already exists at `self.path`.
304
+
305
+ Used to determine whether we can perform an `upsert` (requires existing
306
+ table metadata at `.hoodie/hoodie.properties`) or need to fall back to
307
+ `bulk_insert` for the initial load.
308
+ """
309
+ try:
310
+ self.spark.read.text(f"{self.path}/.hoodie/hoodie.properties").limit(1).collect()
311
+ return True
312
+ except Exception:
313
+ return False
314
+
315
+ def load_hudi(self, df: DataFrame) -> DataFrame:
316
+ # -----------------------------------------------------------------
317
+ # Hudi write operation auto-detection (added 2026-03-02)
318
+ # -----------------------------------------------------------------
319
+ # In Hudi 0.14+, the `upsert` operation expects an existing Hudi table
320
+ # with `.hoodie/hoodie.properties` on HDFS. Writing to a *new* path
321
+ # with `upsert` fails with `HoodieUpsertException`.
322
+ #
323
+ # Previously (Hudi 0.12), `upsert` could auto-create new tables.
324
+ #
325
+ # Decision: auto-detect whether the table exists. If not, override
326
+ # the operation to `bulk_insert` for the initial load. Subsequent
327
+ # writes use the configured `hudi_mode` (typically `upsert`).
328
+ # -----------------------------------------------------------------
329
+ operation = self.hudi_mode
330
+ needs_existing_table = operation in ("upsert", "insert_overwrite", "insert_overwrite_table")
331
+ if needs_existing_table and not self._is_existing_hudi_table():
332
+ logger.info(
333
+ f"Hudi table not found at {self.path}, "
334
+ f"falling back to 'bulk_insert' for initial load "
335
+ f"(configured mode: {operation})"
336
+ )
337
+ operation = "bulk_insert"
338
+
339
+ hudi_options = {
340
+ "hoodie.table.name": f"{self.dst_db}.{self.dst_tbl}",
341
+ "hoodie.datasource.write.table.type": "COPY_ON_WRITE",
342
+ "hoodie.datasource.write.recordkey.field": self.id,
343
+ "hoodie.datasource.write.partitionpath.field": ",".join(self.par_cols),
344
+ "hoodie.datasource.write.keygenerator.class": "org.apache.hudi.keygen.ComplexKeyGenerator",
345
+ "hoodie.datasource.write.precombine.field": self.ts,
346
+ "hoodie.datasource.write.operation": f"{operation}",
347
+ "hoodie.bulkinsert.sort.mode": "PARTITION_PATH_REPARTITION_AND_SORT",
348
+ "hoodie.datasource.write.reconcile.schema": True,
349
+ # Do not set hoodie.*.shuffle.parallelism or spark.sql.shuffle.partitions:
350
+ # each shuffle task becomes a tiny Hudi file.
351
+ "hoodie.schema.on.read.enable": True,
352
+ # ---------------------------------------------------------------
353
+ # Hudi 0.15 ComplexKeyGenerator regression fix
354
+ # ---------------------------------------------------------------
355
+ # Hudi 0.14.1, 0.15.0, 1.0.x introduced a regression in
356
+ # ComplexKeyGenerator with a single record key field - the record
357
+ # key encoding changed, which can cause duplicate records during
358
+ # upserts. Setting this to `true` ensures consistent encoding
359
+ # for tables created on these versions.
360
+ #
361
+ # Ref: https://hudi.apache.org/releases/release-1.1.1
362
+ # ---------------------------------------------------------------
363
+ "hoodie.write.complex.keygen.new.encoding": True,
364
+ "hoodie.datasource.hive_sync.enable": True,
365
+ "hoodie.datasource.hive_sync.create_managed_table": True,
366
+ "hoodie.datasource.hive_sync.database": self.dst_db,
367
+ "hoodie.datasource.hive_sync.table": self.dst_tbl,
368
+ "hoodie.datasource.hive_sync.skip_ro_suffix": False,
369
+ "hoodie.datasource.hive_sync.support_timestamp": True,
370
+ "hoodie.datasource.write.hive_style_partitioning": True,
371
+ "hoodie.metadata.enable": False,
372
+ "hoodie.write.concurrency.mode": f"{self.concurrency_mode}",
373
+ "hoodie.write.lock.zookeeper.base_path": "/hudi",
374
+ "hoodie.write.lock.zookeeper.lock_key": f"{self.dst_db}.{self.dst_tbl}",
375
+ "hoodie.write.lock.zookeeper.port": "2181",
376
+ "hoodie.write.lock.zookeeper.url": f"{self.zookeeper}",
377
+ "path": self.path,
378
+ }
379
+ hudi_options.update(self.extra_hudi_options)
380
+ hudi_options.update(self.hudi_layout.write_options())
381
+ # Applied last, deliberately. A table inside the daily v3 metadata
382
+ # listing scope must have its Hudi metadata index maintained by every
383
+ # writer; an `extra_hudi_options` opt-out would leave the index stale
384
+ # and make readers see the wrong file list. Tables outside the scope
385
+ # keep `hoodie.metadata.enable=false` exactly as before.
386
+ hudi_options.update(
387
+ hudi_metadata_write_options(f"{self.dst_db}.{self.dst_tbl}")
388
+ )
389
+
390
+ df.write.format("hudi").options(**hudi_options).mode("append").save()
391
+
392
+ def rewrite_hudi_layout(self, partition_filter: Optional[str] = None) -> None:
393
+ """Repair small files on an existing Hudi table.
394
+
395
+ Reads the published path, drops `_hoodie_*` meta columns, and
396
+ ``insert_overwrite``s with ``HudiLayout``. Do not set Spark/Hudi
397
+ shuffle parallelism. Pass a Spark SQL predicate to limit the rewrite
398
+ (for example ``dt = '2026-08-01'``).
399
+ """
400
+ if self.table_type != "hudi_table":
401
+ raise ValueError("rewrite_hudi_layout is for hudi_table only")
402
+ if not self._is_existing_hudi_table():
403
+ raise ValueError(f"no Hudi table at {self.path}")
404
+ source = self.spark.read.format("hudi").load(self.path)
405
+ if partition_filter:
406
+ source = source.where(partition_filter)
407
+ meta = [c for c in source.columns if c.startswith("_hoodie_")]
408
+ source = source.drop(*meta) if meta else source
409
+ previous = self.hudi_mode
410
+ try:
411
+ self.hudi_mode = "insert_overwrite"
412
+ self.load_hudi(source)
413
+ finally:
414
+ self.hudi_mode = previous
415
+
416
+ def load_spark(self, df: DataFrame):
417
+ output = df.repartition(*self.repartition) if self.repartition else df
418
+ try:
419
+ output.write.mode("overwrite").insertInto(f"{self.dst_db}.{self.dst_tbl}")
420
+ except AnalysisException:
421
+ writer = output.write.option("path", self.path).mode("overwrite")
422
+ if len(self.par_cols) == 0:
423
+ writer.saveAsTable(f"{self.dst_db}.{self.dst_tbl}")
424
+ else:
425
+ writer.partitionBy(*self.par_cols).saveAsTable(
426
+ f"{self.dst_db}.{self.dst_tbl}"
427
+ )
428
+
429
+ def load(self, df: DataFrame):
430
+ if len(self.par_cols) > 0:
431
+ df = self._prepare_partition(df)
432
+ if self.table_type == "hudi_table":
433
+ self.load_hudi(df)
434
+ elif self.table_type == "spark_table":
435
+ self.load_spark(df)
436
+ else:
437
+ raise ValueError(f'table_type "{self.table_type}" is incorrect')
438
+
439
+ def etl(self, start_date: str = None, end_date: str = None):
440
+ df = self.extract(start_date=start_date, end_date=end_date)
441
+ res = self.transform(df)
442
+ self.load(res)
443
+
444
+ def process(self):
445
+ if self.bulk:
446
+ self.etl(self.start_date, self.end_date)
447
+ else:
448
+ # 1 Batch per day
449
+ logger.info("Processing ETL task on a per day basis")
450
+ date_rng = pd.date_range(self.start_date, self.end_date)
451
+ for dt in date_rng:
452
+ start_dt = dt.strftime("%Y-%m-%d")
453
+ end_dt = dt + relativedelta(days=1)
454
+ end_dt = end_dt.strftime("%Y-%m-%d")
455
+ logger.info(f"Processing dt {start_dt}")
456
+ self.etl(start_dt, end_dt)
457
+
458
+
459
+ class MongoDbEtl(Etl):
460
+ schema = None
461
+
462
+ def extract(
463
+ self, start_date: str = None, end_date: str = None, schema: StructField = None
464
+ ):
465
+ start_date = start_date or self.start_date
466
+ end_date = end_date or self.end_date
467
+ schema = schema or self.schema
468
+ df = (
469
+ self.spark.read.format("mongodb")
470
+ .option("connection.uri", self.url)
471
+ .option("database", self.src_db)
472
+ .option("collection", self.src_tbl)
473
+ .option("sql.inferSchema.mapTypes.enabled", "true")
474
+ .load(schema=schema)
475
+ )
476
+ if start_date:
477
+ df = df.filter(col(self.filter_by) >= start_date)
478
+ if end_date:
479
+ df = df.filter(col(self.filter_by) < end_date)
480
+ return df
481
+
482
+
483
+ class JdbcEtl(Etl):
484
+ driver = None
485
+
486
+ def __init__(
487
+ self,
488
+ url: str,
489
+ start_date: str = None,
490
+ end_date: str = None,
491
+ bulk=True,
492
+ hudi_mode_override: str = None,
493
+ **kwargs,
494
+ ):
495
+ super().__init__(start_date, end_date, url=url, bulk=bulk, hudi_mode_override=hudi_mode_override, **kwargs)
496
+
497
+ @classmethod
498
+ def run_from_cli(
499
+ cls,
500
+ url: Annotated[str, typer.Option("--url")],
501
+ start_date: Annotated[str, typer.Option("--start-date")] = None,
502
+ end_date: Annotated[str, typer.Option("--end-date")] = None,
503
+ bulk: Annotated[bool, typer.Option("--bulk/--per-day")] = True,
504
+ hudi_mode: Annotated[str, typer.Option("--hudi-mode")] = None,
505
+ ):
506
+ obj = cls.__new__(cls)
507
+ obj.__init__(url, start_date, end_date, bulk, hudi_mode_override=hudi_mode)
508
+ obj()
509
+ return obj
510
+
511
+ def _prepare_query(self, start_date: str = None, end_date: str = None):
512
+ query = f"select * from {self.src_db}.{self.src_tbl} where 1 = 1 "
513
+ if self.filter_by and start_date:
514
+ query += f'and {self.filter_by} >= "{start_date}" '
515
+ if self.filter_by and end_date:
516
+ query += f'and {self.filter_by} < "{end_date}" '
517
+ self.query = f"({query}) as tmp"
518
+ return self.query
519
+
520
+ def extract(self, start_date: str = None, end_date: str = None):
521
+ start_date = start_date or self.start_date
522
+ end_date = end_date or self.end_date
523
+ self._prepare_query(start_date, end_date)
524
+ logger.info(self.query)
525
+ if self.filter_by and start_date and end_date:
526
+ df = (
527
+ self.spark.read.format("jdbc")
528
+ .option("driver", self.driver)
529
+ .option("url", self.url)
530
+ .option("dbtable", self.query)
531
+ .option("partitionColumn", self.ts)
532
+ .option("lowerBound", start_date)
533
+ .option("upperBound", end_date)
534
+ .option("numPartitions", "8")
535
+ .option("fetchsize", "1000")
536
+ .option("pushDownPredicate", "true")
537
+ .option("pushDownAggregate", "true")
538
+ .option("pushDownLimit", "true")
539
+ .option("pushDownOffset", "true")
540
+ .load()
541
+ )
542
+ else:
543
+ df = (
544
+ self.spark.read.format("jdbc")
545
+ .option("driver", self.driver)
546
+ .option("url", self.url)
547
+ .option("dbtable", self.query)
548
+ .option("fetchsize", "1000")
549
+ .option("pushDownPredicate", "true")
550
+ .option("pushDownAggregate", "true")
551
+ .option("pushDownLimit", "true")
552
+ .option("pushDownOffset", "true")
553
+ .load()
554
+ )
555
+ return df
556
+
557
+
558
+ class MySqlEtl(JdbcEtl):
559
+ driver = "com.mysql.cj.jdbc.Driver"
560
+
561
+
562
+ class MsSqlEtl(JdbcEtl):
563
+ """
564
+ ETL base class for Microsoft SQL Server sources via JDBC.
565
+
566
+ JDBC URL format:
567
+ jdbc:sqlserver://host:port;databaseName=db;encrypt=false
568
+
569
+ Differences from MySqlEtl:
570
+ - Driver: com.microsoft.sqlserver.jdbc.SQLServerDriver
571
+ - Date literals use single quotes (ANSI SQL)
572
+ - Identifiers use bracket quoting: [schema].[table]
573
+ """
574
+
575
+ driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
576
+
577
+ def _prepare_query(self, start_date: str = None, end_date: str = None):
578
+ # Use bracket-quoted identifiers for MSSQL
579
+ if self.src_db:
580
+ table_ref = f"[{self.src_db}].[{self.src_tbl}]"
581
+ else:
582
+ table_ref = f"[{self.src_tbl}]"
583
+
584
+ query = f"select * from {table_ref} where 1 = 1 "
585
+ if self.filter_by and start_date:
586
+ query += f"and [{self.filter_by}] >= '{start_date}' "
587
+ if self.filter_by and end_date:
588
+ query += f"and [{self.filter_by}] < '{end_date}' "
589
+ self.query = f"({query}) as tmp"
590
+ return self.query
591
+
592
+ def extract(self, start_date: str = None, end_date: str = None):
593
+ """
594
+ MSSQL-specific extract -- always uses single-partition JDBC read.
595
+
596
+ MSSQL servers often redirect partitioned connections to internal
597
+ hostnames (e.g. 'DB197') that Spark workers cannot resolve.
598
+ Avoid `partitionColumn` / `numPartitions` to prevent this.
599
+
600
+ All semicolon-delimited URL properties (user, password, encrypt, etc.)
601
+ are extracted and passed as explicit Spark JDBC `.option()` calls.
602
+ This is required because:
603
+ - MSSQL JDBC driver v10.2+ defaults `encrypt=true`
604
+ - Spark's JDBC Properties object does NOT merge URL-embedded properties
605
+ - All properties must be in the Properties object for the driver
606
+ """
607
+ start_date = start_date or self.start_date
608
+ end_date = end_date or self.end_date
609
+ self._prepare_query(start_date, end_date)
610
+ logger.info(self.query)
611
+
612
+ # Parse MSSQL JDBC URL: jdbc:sqlserver://host:port;key=val;key=val;...
613
+ parts = self.url.split(";")
614
+ base_url = parts[0] # jdbc:sqlserver://host:port
615
+ jdbc_props = {}
616
+ remaining_url_parts = []
617
+ for part in parts[1:]:
618
+ if "=" in part:
619
+ k, v = part.split("=", 1)
620
+ # databaseName must stay in the URL (required by the driver)
621
+ # redirect must stay in the URL (connection-level property;
622
+ # the driver ignores it when passed as a JDBC Properties entry)
623
+ if k.lower() in ("databasename", "redirect"):
624
+ remaining_url_parts.append(part)
625
+ else:
626
+ jdbc_props[k] = v
627
+ else:
628
+ remaining_url_parts.append(part)
629
+
630
+ clean_url = ";".join([base_url] + remaining_url_parts)
631
+
632
+ # Ensure SSL properties are set (driver v10.2+ defaults encrypt=true)
633
+ jdbc_props.setdefault("encrypt", "false")
634
+ jdbc_props.setdefault("trustServerCertificate", "true")
635
+
636
+ logger.info(f"MSSQL clean URL: {clean_url}")
637
+ logger.info(f"MSSQL props keys: {list(jdbc_props.keys())}")
638
+
639
+ df = (
640
+ self.spark.read.format("jdbc")
641
+ .option("driver", self.driver)
642
+ .option("url", clean_url)
643
+ .option("dbtable", self.query)
644
+ .option("fetchsize", "1000")
645
+ .options(**jdbc_props)
646
+ .load()
647
+ )
648
+ return df
649
+
650
+
651
+ class JsonEtl(Etl):
652
+ src_path = None
653
+ schema = None
654
+
655
+ def __init__(
656
+ self,
657
+ src_path: str,
658
+ start_date: str = None,
659
+ end_date: str = None,
660
+ bulk=True,
661
+ recursive: bool = True,
662
+ **kwargs,
663
+ ):
664
+ super().__init__(start_date, end_date, bulk, **kwargs)
665
+ self.src_path = src_path
666
+ self.recursive = recursive
667
+
668
+ @classmethod
669
+ def run_from_cli(
670
+ cls,
671
+ src_path: Annotated[str, typer.Option("--src-path")] = None,
672
+ start_date: Annotated[str, typer.Option("--start-date")] = None,
673
+ end_date: Annotated[str, typer.Option("--end-date")] = None,
674
+ bulk: Annotated[bool, typer.Option("--bulk/--per-day")] = True,
675
+ recursive: Annotated[
676
+ bool, typer.Option("--recursive/--no-recursive")
677
+ ] = True,
678
+ ):
679
+ obj = cls.__new__(cls)
680
+ obj.__init__(src_path, start_date, end_date, bulk, recursive)
681
+ obj()
682
+ return obj
683
+
684
+ def extract(self, start_date: str = None, end_date: str = None):
685
+ start_date = start_date or self.start_date
686
+ end_date = end_date or self.end_date
687
+ recursive = str(self.recursive).lower()
688
+ if self.schema:
689
+ return (
690
+ self.spark.read.option("recursiveFileLookup", recursive)
691
+ .schema(self.schema)
692
+ .json(self.src_path)
693
+ )
694
+ else:
695
+ return self.spark.read.option("recursiveFileLookup", recursive).json(
696
+ self.src_path
697
+ )
698
+
699
+
700
+ class CsvEtl(Etl):
701
+ src_path = None
702
+ schema = None
703
+
704
+ def __init__(
705
+ self,
706
+ src_path: str,
707
+ start_date: str = None,
708
+ end_date: str = None,
709
+ bulk=True,
710
+ recursive: bool = True,
711
+ **kwargs,
712
+ ):
713
+ super().__init__(start_date, end_date, bulk=bulk, **kwargs)
714
+ self.src_path = src_path
715
+ self.recursive = recursive
716
+
717
+ @classmethod
718
+ def run_from_cli(
719
+ cls,
720
+ src_path: Annotated[str, typer.Option("--src-path")] = None,
721
+ start_date: Annotated[str, typer.Option("--start-date")] = None,
722
+ end_date: Annotated[str, typer.Option("--end-date")] = None,
723
+ bulk: Annotated[bool, typer.Option("--bulk/--per-day")] = True,
724
+ recursive: Annotated[
725
+ bool, typer.Option("--recursive/--no-recursive")
726
+ ] = True,
727
+ ):
728
+ obj = cls.__new__(cls)
729
+ obj.__init__(src_path, start_date, end_date, bulk, recursive)
730
+ obj()
731
+ return obj
732
+
733
+ def extract(self, start_date: str = None, end_date: str = None):
734
+ if self.schema:
735
+ return (
736
+ self.spark.read.option("recursiveFileLookup", self.recursive)
737
+ .option("header", True)
738
+ .option("mode", "DROPMALFORMED")
739
+ .schema(self.schema)
740
+ .csv(self.src_path)
741
+ )
742
+ else:
743
+ return (
744
+ self.spark.read.option("recursiveFileLookup", self.recursive)
745
+ .option("header", True)
746
+ .option("mode", "DROPMALFORMED")
747
+ .csv(self.src_path)
748
+ )