@whaly/connector-sdk 0.3.2 → 0.3.4

package/dist/index.mjs

@@ -1550,6 +1550,9 @@ var MissingSchemaError = class extends Error {
   }
 };
 
+// src/sdk/models/target/models.ts
+var DEFAULT_SYNCED_AT_COLUMN = "_wly_synced_at";
+
 // src/sdk/models/target/record.ts
 var removeParasiteProperties = (record, schema) => {
   const schemaKeys = Object.keys(schema);
@@ -1560,10 +1563,10 @@ var removeParasiteProperties = (record, schema) => {
   });
   return record;
 };
-var addWhalyFields = (record, batchDate) => {
+var addWhalyFields = (record, batchDate, columnName = DEFAULT_SYNCED_AT_COLUMN) => {
   return {
     ...record,
-    _whaly_synced: batchDate
+    [columnName]: batchDate
   };
 };
 
@@ -1832,6 +1835,8 @@ import Bluebird3 from "bluebird";
 var semaphore = new Semaphore(1);
 var ITarget = class _ITarget {
   config;
+  syncedAtColumnName;
+  syncedAtColumnUseLegacyStringType;
   schemaHooks;
   syncTime;
   stateProvider;
@@ -1847,6 +1852,8 @@ var ITarget = class _ITarget {
   ajv;
   constructor(config, stateProvider) {
     this.config = config;
+    this.syncedAtColumnName = config.syncedAtColumnName ?? DEFAULT_SYNCED_AT_COLUMN;
+    this.syncedAtColumnUseLegacyStringType = config.syncedAtColumnUseLegacyStringType ?? false;
     this.stateProvider = stateProvider;
     this.schemaHooks = [];
     this.streams = {};
@@ -1867,8 +1874,8 @@ var ITarget = class _ITarget {
   analyzeSchemaChanges = (previousSchema, newSchema, streamId) => {
     const changes = [];
     let hasBreakingChanges = false;
-    const prevColumns = new Set(Object.keys(previousSchema).filter((col) => !col.startsWith("_whaly")));
-    const newColumns = new Set(Object.keys(newSchema).filter((col) => !col.startsWith("_whaly")));
+    const prevColumns = new Set(Object.keys(previousSchema).filter((col) => col !== this.syncedAtColumnName));
+    const newColumns = new Set(Object.keys(newSchema).filter((col) => col !== this.syncedAtColumnName));
     for (const columnName of newColumns) {
       if (!prevColumns.has(columnName)) {
         changes.push(`Added column '${columnName}'`);
@@ -1926,7 +1933,7 @@ var ITarget = class _ITarget {
         return;
       }
       const recordWithoutParasiteProperties = removeParasiteProperties(record, schema);
-      const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate());
+      const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate(), this.syncedAtColumnName);
       const validateFn = stream.getCompiledValidateFn();
       if (validateFn && !validateFn(recordWithWhalyFields)) {
         throw new SchemaValidationError(`Stream: ${streamId} - Record is not valid according to schema.
@@ -2009,10 +2016,7 @@ var ITarget = class _ITarget {
         throw new MissingSchemaError(`Stream: ${streamId} - Received SCHEMA message before stream state was initialized.`, streamId);
       }
       const schema = flattenSchema(streamId, message.schema);
-      schema["_whaly_synced"] = {
-        format: "date-time",
-        type: ["null", "string"]
-      };
+      schema[this.syncedAtColumnName] = this.syncedAtColumnUseLegacyStringType ? { type: ["null", "string"] } : { format: "date-time", type: ["null", "string"] };
       const dbSync = streamState.getDbSync();
       Object.keys(schema).forEach((fieldUnsafeKey) => {
         const fieldDef = schema[fieldUnsafeKey];
@@ -2514,7 +2518,6 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
     csvOptions.headers = false;
   }
   const csvStream = csv(csvOptions);
-  const syncedAt = (/* @__PURE__ */ new Date()).toISOString();
   let initialReadStream;
   let finalInputStream;
   const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
@@ -2529,7 +2532,7 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
   try {
     for await (const row of Readable.from(csvStream)) {
       if (!isGeneratorConfigArray) {
-        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        const rowData = {};
         for (const key of configKeys) {
           const keyGenerator = fileConfig.fields[key.trim()];
           if (!keyGenerator) continue;
@@ -2539,7 +2542,7 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
         yield rowData;
       } else {
         const rowKeys = Object.keys(row);
-        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        const rowData = {};
         for (let i = 0; i < rowKeys.length; i++) {
           const keyGenerator = fileConfig.fields[i];
           if (!keyGenerator) continue;
@@ -2842,7 +2845,6 @@ async function processFileStreams(entries, tapState, target) {
 var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
   _separator = ",";
   _encoding;
-  _addSyncedAtColumn;
   _fields;
   _fieldsMode;
   _replicationMethod;
@@ -2858,10 +2860,6 @@ var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
     this._encoding = enc;
     return this;
   }
-  addSyncedAtColumn(enabled = true) {
-    this._addSyncedAtColumn = enabled;
-    return this;
-  }
   fieldsFromDict(mapping) {
     if (this._fieldsMode === "array") {
       throw new Error("Cannot use fieldsFromDict after fieldsFromArray");
@@ -2903,9 +2901,6 @@ var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
     if (this._encoding !== void 0) {
       config.encoding = this._encoding;
     }
-    if (this._addSyncedAtColumn !== void 0) {
-      config.addSyncedAtColumn = this._addSyncedAtColumn;
-    }
     return config;
   }
   buildStreamConfig(streamId) {
@@ -3791,6 +3786,7 @@ export {
   CounterMetric,
   CsvExtractionConfigBuilder,
   DEFAULT_MAX_CONCURRENT_STREAMS,
+  DEFAULT_SYNCED_AT_COLUMN,
   EXECUTION_TIME_METRIC_NAME,
   ExcelExtractionConfigBuilder,
   FileFormat,