@whaly/connector-sdk 0.3.1 → 0.3.3

package/dist/index.mjs

@@ -1550,6 +1550,9 @@ var MissingSchemaError = class extends Error {
   }
 };
 
+// src/sdk/models/target/models.ts
+var DEFAULT_SYNCED_AT_COLUMN = "_wly_synced_at";
+
 // src/sdk/models/target/record.ts
 var removeParasiteProperties = (record, schema) => {
   const schemaKeys = Object.keys(schema);
@@ -1560,10 +1563,10 @@ var removeParasiteProperties = (record, schema) => {
   });
   return record;
 };
-var addWhalyFields = (record, batchDate) => {
+var addWhalyFields = (record, batchDate, columnName = DEFAULT_SYNCED_AT_COLUMN) => {
   return {
     ...record,
-    _whaly_synced: batchDate
+    [columnName]: batchDate
   };
 };
 
@@ -1832,6 +1835,7 @@ import Bluebird3 from "bluebird";
 var semaphore = new Semaphore(1);
 var ITarget = class _ITarget {
   config;
+  syncedAtColumnName;
   schemaHooks;
   syncTime;
   stateProvider;
@@ -1847,6 +1851,7 @@ var ITarget = class _ITarget {
   ajv;
   constructor(config, stateProvider) {
     this.config = config;
+    this.syncedAtColumnName = config.syncedAtColumnName ?? DEFAULT_SYNCED_AT_COLUMN;
     this.stateProvider = stateProvider;
     this.schemaHooks = [];
     this.streams = {};
@@ -1867,8 +1872,8 @@ var ITarget = class _ITarget {
   analyzeSchemaChanges = (previousSchema, newSchema, streamId) => {
     const changes = [];
     let hasBreakingChanges = false;
-    const prevColumns = new Set(Object.keys(previousSchema).filter((col) => !col.startsWith("_whaly")));
-    const newColumns = new Set(Object.keys(newSchema).filter((col) => !col.startsWith("_whaly")));
+    const prevColumns = new Set(Object.keys(previousSchema).filter((col) => col !== this.syncedAtColumnName));
+    const newColumns = new Set(Object.keys(newSchema).filter((col) => col !== this.syncedAtColumnName));
     for (const columnName of newColumns) {
       if (!prevColumns.has(columnName)) {
         changes.push(`Added column '${columnName}'`);
@@ -1926,7 +1931,7 @@ var ITarget = class _ITarget {
         return;
       }
       const recordWithoutParasiteProperties = removeParasiteProperties(record, schema);
-      const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate());
+      const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate(), this.syncedAtColumnName);
       const validateFn = stream.getCompiledValidateFn();
       if (validateFn && !validateFn(recordWithWhalyFields)) {
         throw new SchemaValidationError(`Stream: ${streamId} - Record is not valid according to schema.
@@ -2009,7 +2014,7 @@ var ITarget = class _ITarget {
         throw new MissingSchemaError(`Stream: ${streamId} - Received SCHEMA message before stream state was initialized.`, streamId);
       }
       const schema = flattenSchema(streamId, message.schema);
-      schema["_whaly_synced"] = {
+      schema[this.syncedAtColumnName] = {
         format: "date-time",
         type: ["null", "string"]
       };
@@ -2305,10 +2310,10 @@ var extractSingleSheetRows = async (fileName, conf) => {
         if (!variables) {
           throw new Error(`No variables extracted from filename, cannot extract derived field ${key}`);
         }
-        acc[key] = variables[column.variableName] || "";
+        acc[key] = variables[column.variableName] || null;
       } else {
         const colIndex = excelColumnToIndex(column.column);
-        acc[key] = row[colIndex]?.v || "";
+        acc[key] = row[colIndex]?.v || null;
       }
       return acc;
     }, {});
@@ -2514,7 +2519,6 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
     csvOptions.headers = false;
   }
   const csvStream = csv(csvOptions);
-  const syncedAt = (/* @__PURE__ */ new Date()).toISOString();
   let initialReadStream;
   let finalInputStream;
   const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
@@ -2529,20 +2533,22 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
   try {
     for await (const row of Readable.from(csvStream)) {
       if (!isGeneratorConfigArray) {
-        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        const rowData = {};
         for (const key of configKeys) {
           const keyGenerator = fileConfig.fields[key.trim()];
           if (!keyGenerator) continue;
-          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key];
+          const rawValue = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key];
+          rowData[keyGenerator.key] = rawValue === "" ? null : rawValue;
         }
         yield rowData;
       } else {
         const rowKeys = Object.keys(row);
-        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        const rowData = {};
         for (let i = 0; i < rowKeys.length; i++) {
           const keyGenerator = fileConfig.fields[i];
           if (!keyGenerator) continue;
-          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i];
+          const rawValue = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i];
+          rowData[keyGenerator.key] = rawValue === "" ? null : rawValue;
         }
         yield rowData;
       }
@@ -2840,7 +2846,6 @@ async function processFileStreams(entries, tapState, target) {
 var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
   _separator = ",";
   _encoding;
-  _addSyncedAtColumn;
   _fields;
   _fieldsMode;
   _replicationMethod;
@@ -2856,10 +2861,6 @@ var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
     this._encoding = enc;
     return this;
   }
-  addSyncedAtColumn(enabled = true) {
-    this._addSyncedAtColumn = enabled;
-    return this;
-  }
   fieldsFromDict(mapping) {
     if (this._fieldsMode === "array") {
       throw new Error("Cannot use fieldsFromDict after fieldsFromArray");
@@ -2901,9 +2902,6 @@ var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
     if (this._encoding !== void 0) {
       config.encoding = this._encoding;
     }
-    if (this._addSyncedAtColumn !== void 0) {
-      config.addSyncedAtColumn = this._addSyncedAtColumn;
-    }
     return config;
   }
   buildStreamConfig(streamId) {
@@ -3789,6 +3787,7 @@ export {
   CounterMetric,
   CsvExtractionConfigBuilder,
   DEFAULT_MAX_CONCURRENT_STREAMS,
+  DEFAULT_SYNCED_AT_COLUMN,
   EXECUTION_TIME_METRIC_NAME,
   ExcelExtractionConfigBuilder,
   FileFormat,