@whaly/connector-sdk 0.3.6 → 0.3.7

package/dist/index.js

@@ -3113,26 +3113,41 @@ var FileStream = class extends Stream {
    */
   async *_getRecords() {
     let total = 0;
+    const cachedExcelData = [];
+    let schemaEmitted = await this.getSchema() !== void 0;
     for (const filePath of this.localFilePaths) {
       if (this.config.format === "EXCEL" /* EXCEL */) {
         const data = await extractExcelRows(filePath, this.config.excel);
+        cachedExcelData.push(data);
         total += data.length;
+        if (!schemaEmitted && data.length > 0) {
+          const properties = Object.fromEntries(
+            Object.keys(data[0]).map((k) => [k, { type: ["null", "string"] }])
+          );
+          await this.target.schema(new SchemaMessage({
+            stream: this.streamId,
+            schema: { type: "object", properties },
+            keyProperties: this.primaryKey
+          }));
+          schemaEmitted = true;
+        }
       } else if (this.config.format === "CSV" /* CSV */) {
         total += await countCsvLines(filePath);
       }
     }
     this.totalRows = total;
-    for (const filePath of this.localFilePaths) {
-      if (this.config.format === "EXCEL" /* EXCEL */) {
-        const data = await extractExcelRows(filePath, this.config.excel);
+    if (this.config.format === "EXCEL" /* EXCEL */) {
+      for (const data of cachedExcelData) {
         for (const row of data) {
           yield row;
         }
-      } else if (this.config.format === "CSV" /* CSV */) {
+      }
+    } else if (this.config.format === "CSV" /* CSV */) {
+      for (const filePath of this.localFilePaths) {
         yield* rowGeneratorFromCsv(filePath, this.config.csv);
-      } else {
-        throw new Error(`FileStream: Unsupported format for stream ${this.streamId}`);
       }
+    } else {
+      throw new Error(`FileStream: Unsupported format for stream ${this.streamId}`);
     }
   }
 };