@whaly/connector-sdk 0.3.5 → 0.3.7

package/dist/index.js

@@ -1274,6 +1274,7 @@ var Stream = class {
   // Private sync methods:
   async _syncRecords(parent) {
     let rowsSent = 0;
+    const dryRunLimit = isDryRun() ? getDryRunLimit() : void 0;
     let recordSyncedMetrics = [];
     if (this.rowsSyncedMetricsConf.isEnabled() === true) {
       recordSyncedMetrics = getCounterMetrics(
@@ -1315,6 +1316,10 @@ var Stream = class {
         metric.increment();
       });
       rowsSent += 1;
+      if (dryRunLimit !== void 0 && rowsSent >= dryRunLimit) {
+        logger.info(`\u{1F6D1} Stream: ${this.streamId} - DRY_RUN_LIMIT reached (${dryRunLimit} records), stopping.`);
+        break;
+      }
       if (rowsSent % 1e3 === 0) {
         const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
         logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
@@ -2340,11 +2345,12 @@ var ITarget = class _ITarget {
       streamState.setSchema(schema);
       streamState.setCompiledValidateFn(this.ajv.compile({ type: "object", properties: schema }));
       const streamReplicationMethod = this.streams[streamId]?.getReplicationMethod();
-      if (message.keyProperties.length === 0 && streamReplicationMethod !== "APPEND" /* APPEND */) {
+      if (message.keyProperties.length === 0 && streamReplicationMethod === "INCREMENTAL" /* INCREMENTAL */) {
         throw new Error(`StreamId: ${message.stream} - \`key_properties\` field is required and can't be empty in SCHEMA message.
             Received SCHEMA message: ${JSON.stringify(message)}`);
       }
       streamState.setKeyProperties(message.keyProperties);
+      dbSync.renamedColumnStore.computeColumnNameForStream(streamId, schema);
       if (!isDryRun()) {
         await dbSync.updateSchemaInWarehouse(schema, message.keyProperties);
         const {
@@ -3107,26 +3113,41 @@ var FileStream = class extends Stream {
    */
   async *_getRecords() {
     let total = 0;
+    const cachedExcelData = [];
+    let schemaEmitted = await this.getSchema() !== void 0;
     for (const filePath of this.localFilePaths) {
       if (this.config.format === "EXCEL" /* EXCEL */) {
         const data = await extractExcelRows(filePath, this.config.excel);
+        cachedExcelData.push(data);
         total += data.length;
+        if (!schemaEmitted && data.length > 0) {
+          const properties = Object.fromEntries(
+            Object.keys(data[0]).map((k) => [k, { type: ["null", "string"] }])
+          );
+          await this.target.schema(new SchemaMessage({
+            stream: this.streamId,
+            schema: { type: "object", properties },
+            keyProperties: this.primaryKey
+          }));
+          schemaEmitted = true;
+        }
       } else if (this.config.format === "CSV" /* CSV */) {
         total += await countCsvLines(filePath);
       }
     }
     this.totalRows = total;
-    for (const filePath of this.localFilePaths) {
-      if (this.config.format === "EXCEL" /* EXCEL */) {
-        const data = await extractExcelRows(filePath, this.config.excel);
+    if (this.config.format === "EXCEL" /* EXCEL */) {
+      for (const data of cachedExcelData) {
         for (const row of data) {
           yield row;
         }
-      } else if (this.config.format === "CSV" /* CSV */) {
+      }
+    } else if (this.config.format === "CSV" /* CSV */) {
+      for (const filePath of this.localFilePaths) {
         yield* rowGeneratorFromCsv(filePath, this.config.csv);
-      } else {
-        throw new Error(`FileStream: Unsupported format for stream ${this.streamId}`);
       }
+    } else {
+      throw new Error(`FileStream: Unsupported format for stream ${this.streamId}`);
     }
   }
 };
@@ -4223,6 +4244,12 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
     });
   };
   getReplaceQueries = () => {
+    if (this.primaryKeys.length === 0) {
+      return [
+        `TRUNCATE TABLE ${this.sqlTableId};`,
+        ...this.getAppendQueries()
+      ];
+    }
     return [
       `TRUNCATE TABLE ${this.sqlTableId};`,
       ...this.getMergeQueries()