@whaly/connector-sdk 0.3.0 → 0.3.2

package/dist/index.mjs

@@ -728,6 +728,8 @@ var Stream = class {
   // However, a STATE will still be emitted if the Stream is using an incremental sync type.
   isSilent = false;
   childConcurrency;
+  // Optional total row count for percentage progress logging
+  totalRows;
   // Metrics configuration
   rowsSyncedMetricsConf;
   executionTimeMetricsConf;
@@ -1005,6 +1007,10 @@ var Stream = class {
         metric.increment();
       });
       rowsSent += 1;
+      if (rowsSent % 1e3 === 0) {
+        const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
+        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
+      }
     }
     const stateServiceInst = StateService.getInstance();
     const state = stateServiceInst.getBookmark(this.streamId);
@@ -1677,7 +1683,8 @@ var flattenSchema = (streamId, schema) => {
 };
 
 // src/sdk/models/target/target.ts
-import { Validator } from "jsonschema";
+import Ajv from "ajv";
+import addFormats from "ajv-formats";
 import dayjs4 from "dayjs";
 
 // src/sdk/models/target/streamDbState.ts
@@ -1755,6 +1762,8 @@ var StreamState = class {
   // So we need to keep track of whether we are at the first load or not
   _hasBeenLoadedYetDuringThisSync;
   replicationMethod;
+  // Pre-compiled ajv ValidateFunction — compiled once per schema, reused for every row
+  compiledValidateFn;
   constructor(streamId, dbSync, replicationMethod) {
     this.streamId = streamId;
     this.dbSync = dbSync;
@@ -1772,6 +1781,12 @@ var StreamState = class {
   getSchema() {
     return this.schema;
   }
+  setCompiledValidateFn(fn) {
+    this.compiledValidateFn = fn;
+  }
+  getCompiledValidateFn() {
+    return this.compiledValidateFn;
+  }
   getBatchDate() {
     return this.batchDate.format(defaultDateTimeFormat);
   }
@@ -1828,8 +1843,8 @@ var ITarget = class _ITarget {
   flushedState;
   renameColumnStore;
   streams;
-  // JSON Schema validator
-  validator;
+  // JSON Schema validator (ajv — pre-compiles schemas into optimized JS functions)
+  ajv;
   constructor(config, stateProvider) {
     this.config = config;
     this.stateProvider = stateProvider;
@@ -1837,7 +1852,7 @@ var ITarget = class _ITarget {
     this.streams = {};
     this.batchedState = {};
     this.flushedState = {};
-    this.validator = new Validator();
+    this.ajv = addFormats(new Ajv({ allErrors: true, strict: false }));
     this.syncTime = dayjs4();
     this.renameColumnStore = new RenameColumnStore();
   }
@@ -1896,12 +1911,14 @@ var ITarget = class _ITarget {
       if (!stream) {
         throw new MissingSchemaError(`A record for stream:\`${streamId}\` was encountered before a corresponding schema.`, streamId);
       }
-      await semaphore.runExclusive(async () => {
-        const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
-        if (shouldUploadIntermediateBatch) {
-          await this.loadAllStreamsInWarehouse({ isFinalLoad: false });
-        }
-      });
+      if (stream.getBatchedRowCount() % 1e4 === 0) {
+        await semaphore.runExclusive(async () => {
+          const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
+          if (shouldUploadIntermediateBatch) {
+            await this.loadAllStreamsInWarehouse({ isFinalLoad: false });
+          }
+        });
+      }
       const record = message.record;
       const schema = stream.getSchema();
       if (!schema) {
@@ -1910,13 +1927,13 @@ var ITarget = class _ITarget {
       }
       const recordWithoutParasiteProperties = removeParasiteProperties(record, schema);
       const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate());
-      const validationResult = this.validator.validate(recordWithWhalyFields, schema);
-      if (!validationResult.valid) {
+      const validateFn = stream.getCompiledValidateFn();
+      if (validateFn && !validateFn(recordWithWhalyFields)) {
         throw new SchemaValidationError(`Stream: ${streamId} - Record is not valid according to schema.
-                Validation errors: ${JSON.stringify(validationResult.errors)}
+                Validation errors: ${JSON.stringify(validateFn.errors)}
 
                 Record: ${JSON.stringify(recordWithoutParasiteProperties)}
-                Schema: ${JSON.stringify(stream.getSchema())}`, streamId, validationResult.errors);
+                Schema: ${JSON.stringify(stream.getSchema())}`, streamId, validateFn.errors ?? []);
       }
       const recordWithValidDate = this.validateDateRange(recordWithWhalyFields, schema);
       const recordWithDecimal = this.convertNumberIntoDecimal(recordWithValidDate, schema);
@@ -2005,6 +2022,7 @@ var ITarget = class _ITarget {
         }
       });
       streamState.setSchema(schema);
+      streamState.setCompiledValidateFn(this.ajv.compile({ type: "object", properties: schema }));
       const streamReplicationMethod = this.streams[streamId]?.getReplicationMethod();
       if (message.keyProperties.length === 0 && streamReplicationMethod !== "APPEND" /* APPEND */) {
         throw new Error(`StreamId: ${message.stream} - \`key_properties\` field is required and can't be empty in SCHEMA message.
@@ -2287,10 +2305,10 @@ var extractSingleSheetRows = async (fileName, conf) => {
         if (!variables) {
           throw new Error(`No variables extracted from filename, cannot extract derived field ${key}`);
         }
-        acc[key] = variables[column.variableName] || "";
+        acc[key] = variables[column.variableName] || null;
       } else {
         const colIndex = excelColumnToIndex(column.column);
-        acc[key] = row[colIndex]?.v || "";
+        acc[key] = row[colIndex]?.v || null;
       }
       return acc;
     }, {});
@@ -2490,6 +2508,7 @@ Actual:   "${actual}" (${actualInfo.length} chars) [${actualInfo.hex}]`;
 };
 async function* rowGeneratorFromCsv(path2, fileConfig) {
   const isGeneratorConfigArray = Array.isArray(fileConfig.fields);
+  const configKeys = isGeneratorConfigArray ? [] : Object.keys(fileConfig.fields);
   const csvOptions = { separator: fileConfig.separator };
   if (isGeneratorConfigArray) {
     csvOptions.headers = false;
@@ -2510,28 +2529,23 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
   try {
     for await (const row of Readable.from(csvStream)) {
       if (!isGeneratorConfigArray) {
-        const rowData = Object.keys(fileConfig.fields).reduce((acc, key) => {
+        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        for (const key of configKeys) {
           const keyGenerator = fileConfig.fields[key.trim()];
-          if (!keyGenerator) {
-            return acc;
-          }
-          return {
-            ...acc,
-            [keyGenerator.key]: keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key]
-          };
-        }, fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {});
+          if (!keyGenerator) continue;
+          const rawValue = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key];
+          rowData[keyGenerator.key] = rawValue === "" ? null : rawValue;
+        }
         yield rowData;
       } else {
-        const rowData = Object.keys(row).reduce((acc, key, i) => {
+        const rowKeys = Object.keys(row);
+        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        for (let i = 0; i < rowKeys.length; i++) {
           const keyGenerator = fileConfig.fields[i];
-          if (!keyGenerator) {
-            return acc;
-          }
-          return {
-            ...acc,
-            [keyGenerator.key]: keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i]
-          };
-        }, fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {});
+          if (!keyGenerator) continue;
+          const rawValue = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i];
+          rowData[keyGenerator.key] = rawValue === "" ? null : rawValue;
+        }
         yield rowData;
       }
     }
@@ -2539,6 +2553,17 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
     initialReadStream.destroy();
   }
 }
+async function countCsvLines(filePath) {
+  return new Promise((resolve, reject) => {
+    let newlines = 0;
+    createReadStream(filePath).on("data", (chunk2) => {
+      const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
+      for (let i = 0; i < buf.length; i++) {
+        if (buf[i] === 10) newlines++;
+      }
+    }).on("end", () => resolve(Math.max(0, newlines - 1))).on("error", reject);
+  });
+}
 var checkCsvHeaderRow = async (path2, fileConfig) => {
   return new Promise((resolve, reject) => {
     const colsWithParsingConfig = Object.keys(fileConfig.fields).map((key) => key.trim());
@@ -2732,8 +2757,19 @@ var FileStream = class extends Stream {
   /**
    * Yield records from all file(s).
    * When multiple file paths are provided, records are yielded sequentially from each file.
+   * Pre-counts total rows across all files to enable percentage progress logging.
    */
   async *_getRecords() {
+    let total = 0;
+    for (const filePath of this.localFilePaths) {
+      if (this.config.format === "EXCEL" /* EXCEL */) {
+        const data = await extractExcelRows(filePath, this.config.excel);
+        total += data.length;
+      } else if (this.config.format === "CSV" /* CSV */) {
+        total += await countCsvLines(filePath);
+      }
+    }
+    this.totalRows = total;
     for (const filePath of this.localFilePaths) {
       if (this.config.format === "EXCEL" /* EXCEL */) {
         const data = await extractExcelRows(filePath, this.config.excel);
@@ -2982,60 +3018,7 @@ import { Storage } from "@google-cloud/storage";
 import { format as format6 } from "util";
 import * as pathModule from "path";
 import { existsSync, mkdirSync, unlinkSync as unlinkSync2 } from "fs";
-
-// node_modules/uuid/dist/esm-node/rng.js
-import crypto from "crypto";
-var rnds8Pool = new Uint8Array(256);
-var poolPtr = rnds8Pool.length;
-function rng() {
-  if (poolPtr > rnds8Pool.length - 16) {
-    crypto.randomFillSync(rnds8Pool);
-    poolPtr = 0;
-  }
-  return rnds8Pool.slice(poolPtr, poolPtr += 16);
-}
-
-// node_modules/uuid/dist/esm-node/regex.js
-var regex_default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;
-
-// node_modules/uuid/dist/esm-node/validate.js
-function validate(uuid) {
-  return typeof uuid === "string" && regex_default.test(uuid);
-}
-var validate_default = validate;
-
-// node_modules/uuid/dist/esm-node/stringify.js
-var byteToHex = [];
-for (let i = 0; i < 256; ++i) {
-  byteToHex.push((i + 256).toString(16).substr(1));
-}
-function stringify3(arr, offset = 0) {
-  const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
-  if (!validate_default(uuid)) {
-    throw TypeError("Stringified UUID is invalid");
-  }
-  return uuid;
-}
-var stringify_default = stringify3;
-
-// node_modules/uuid/dist/esm-node/v4.js
-function v4(options, buf, offset) {
-  options = options || {};
-  const rnds = options.random || (options.rng || rng)();
-  rnds[6] = rnds[6] & 15 | 64;
-  rnds[8] = rnds[8] & 63 | 128;
-  if (buf) {
-    offset = offset || 0;
-    for (let i = 0; i < 16; ++i) {
-      buf[offset + i] = rnds[i];
-    }
-    return buf;
-  }
-  return stringify_default(rnds);
-}
-var v4_default = v4;
-
-// src/services/cloud-storage.ts
+import { randomUUID } from "crypto";
 var logPrefix3 = "[CloudStorageService]";
 var tmpDir = "tmp";
 var CloudStorageService = class {
@@ -3173,12 +3156,15 @@ var CloudStorageService = class {
   }
   /**
    * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+   * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
+   * or `<prefix>/default/` otherwise.
    * Returns the GCS File reference.
    */
   async uploadFileWithUniqueName(filePath, prefix, streamId) {
     try {
       await this.bucket.get({ autoCreate: false });
-      const destinationFileName = `${prefix}/${streamId}-${v4_default()}.jsonnl`;
+      const runFolder = process.env.RUN_ID ?? "default";
+      const destinationFileName = `${prefix}/${runFolder}/${streamId}-${randomUUID()}.jsonnl`;
       await this.bucket.upload(filePath, {
         destination: destinationFileName
       });
@@ -3834,6 +3820,7 @@ export {
   _greaterThan,
   addWhalyFields,
   checkCsvHeaderRow,
+  countCsvLines,
   createCellReference,
   createCsvStreamConfig,
   createExcelGenerator,