@objectstack/runtime 3.2.2 → 3.2.3

package/dist/index.js

@@ -75,6 +75,518 @@ var DriverPlugin = class {
   }
 };
 
+// src/seed-loader.ts
+var DEFAULT_EXTERNAL_ID_FIELD = "name";
+var SeedLoaderService = class {
+  constructor(engine, metadata, logger) {
+    this.engine = engine;
+    this.metadata = metadata;
+    this.logger = logger;
+  }
+  // ==========================================================================
+  // Public API
+  // ==========================================================================
+  async load(request) {
+    const startTime = Date.now();
+    const config = request.config;
+    const allErrors = [];
+    const allResults = [];
+    const datasets = this.filterByEnv(request.datasets, config.env);
+    if (datasets.length === 0) {
+      return this.buildEmptyResult(config, Date.now() - startTime);
+    }
+    const objectNames = datasets.map((d) => d.object);
+    const graph = await this.buildDependencyGraph(objectNames);
+    this.logger.info("[SeedLoader] Dependency graph built", {
+      objects: objectNames.length,
+      insertOrder: graph.insertOrder,
+      circularDeps: graph.circularDependencies.length
+    });
+    const orderedDatasets = this.orderDatasets(datasets, graph.insertOrder);
+    const refMap = this.buildReferenceMap(graph);
+    const insertedRecords = /* @__PURE__ */ new Map();
+    const deferredUpdates = [];
+    for (const dataset of orderedDatasets) {
+      const result = await this.loadDataset(
+        dataset,
+        config,
+        refMap,
+        insertedRecords,
+        deferredUpdates,
+        allErrors
+      );
+      allResults.push(result);
+      if (config.haltOnError && result.errored > 0) {
+        this.logger.warn("[SeedLoader] Halting on first error", { object: dataset.object });
+        break;
+      }
+    }
+    if (config.multiPass && deferredUpdates.length > 0 && !config.dryRun) {
+      this.logger.info("[SeedLoader] Pass 2: resolving deferred references", {
+        count: deferredUpdates.length
+      });
+      await this.resolveDeferredUpdates(deferredUpdates, insertedRecords, allResults, allErrors);
+    }
+    const durationMs = Date.now() - startTime;
+    return this.buildResult(config, graph, allResults, allErrors, durationMs);
+  }
+  async buildDependencyGraph(objectNames) {
+    const nodes = [];
+    const objectSet = new Set(objectNames);
+    for (const objectName of objectNames) {
+      const objDef = await this.metadata.getObject(objectName);
+      const dependsOn = [];
+      const references = [];
+      if (objDef && objDef.fields) {
+        const fields = objDef.fields;
+        for (const [fieldName, fieldDef] of Object.entries(fields)) {
+          if ((fieldDef.type === "lookup" || fieldDef.type === "master_detail") && fieldDef.reference) {
+            const targetObject = fieldDef.reference;
+            if (objectSet.has(targetObject) && !dependsOn.includes(targetObject)) {
+              dependsOn.push(targetObject);
+            }
+            references.push({
+              field: fieldName,
+              targetObject,
+              targetField: DEFAULT_EXTERNAL_ID_FIELD,
+              fieldType: fieldDef.type
+            });
+          }
+        }
+      }
+      nodes.push({ object: objectName, dependsOn, references });
+    }
+    const { insertOrder, circularDependencies } = this.topologicalSort(nodes);
+    return { nodes, insertOrder, circularDependencies };
+  }
+  async validate(datasets, config) {
+    const { SeedLoaderConfigSchema } = await import("@objectstack/spec/data");
+    const parsedConfig = SeedLoaderConfigSchema.parse({ ...config, dryRun: true });
+    return this.load({ datasets, config: parsedConfig });
+  }
+  // ==========================================================================
+  // Internal: Dataset Loading
+  // ==========================================================================
+  async loadDataset(dataset, config, refMap, insertedRecords, deferredUpdates, allErrors) {
+    const objectName = dataset.object;
+    const mode = dataset.mode || config.defaultMode;
+    const externalId = dataset.externalId || "name";
+    let inserted = 0;
+    let updated = 0;
+    let skipped = 0;
+    let errored = 0;
+    let referencesResolved = 0;
+    let referencesDeferred = 0;
+    const errors = [];
+    if (!insertedRecords.has(objectName)) {
+      insertedRecords.set(objectName, /* @__PURE__ */ new Map());
+    }
+    let existingRecords;
+    if ((mode === "upsert" || mode === "update" || mode === "ignore") && !config.dryRun) {
+      existingRecords = await this.loadExistingRecords(objectName, externalId);
+    }
+    const objectRefs = refMap.get(objectName) || [];
+    for (let i = 0; i < dataset.records.length; i++) {
+      const record = { ...dataset.records[i] };
+      for (const ref of objectRefs) {
+        const fieldValue = record[ref.field];
+        if (fieldValue === void 0 || fieldValue === null) continue;
+        if (typeof fieldValue !== "string" || this.looksLikeInternalId(fieldValue)) continue;
+        const targetMap = insertedRecords.get(ref.targetObject);
+        const resolvedId = targetMap?.get(String(fieldValue));
+        if (resolvedId) {
+          record[ref.field] = resolvedId;
+          referencesResolved++;
+        } else if (!config.dryRun) {
+          const dbId = await this.resolveFromDatabase(ref.targetObject, ref.targetField, fieldValue);
+          if (dbId) {
+            record[ref.field] = dbId;
+            referencesResolved++;
+          } else if (config.multiPass) {
+            record[ref.field] = null;
+            deferredUpdates.push({
+              objectName,
+              recordExternalId: String(record[externalId] ?? ""),
+              field: ref.field,
+              targetObject: ref.targetObject,
+              targetField: ref.targetField,
+              attemptedValue: fieldValue,
+              recordIndex: i
+            });
+            referencesDeferred++;
+          } else {
+            const error = {
+              sourceObject: objectName,
+              field: ref.field,
+              targetObject: ref.targetObject,
+              targetField: ref.targetField,
+              attemptedValue: fieldValue,
+              recordIndex: i,
+              message: `Cannot resolve reference: ${objectName}.${ref.field} = '${fieldValue}' \u2192 ${ref.targetObject}.${ref.targetField} not found`
+            };
+            errors.push(error);
+            allErrors.push(error);
+          }
+        } else {
+          const targetMap2 = insertedRecords.get(ref.targetObject);
+          if (!targetMap2?.has(String(fieldValue))) {
+            const error = {
+              sourceObject: objectName,
+              field: ref.field,
+              targetObject: ref.targetObject,
+              targetField: ref.targetField,
+              attemptedValue: fieldValue,
+              recordIndex: i,
+              message: `[dry-run] Reference may not resolve: ${objectName}.${ref.field} = '${fieldValue}' \u2192 ${ref.targetObject}.${ref.targetField}`
+            };
+            errors.push(error);
+            allErrors.push(error);
+          }
+        }
+      }
+      if (!config.dryRun) {
+        try {
+          const result = await this.writeRecord(
+            objectName,
+            record,
+            mode,
+            externalId,
+            existingRecords
+          );
+          if (result.action === "inserted") inserted++;
+          else if (result.action === "updated") updated++;
+          else if (result.action === "skipped") skipped++;
+          const externalIdValue = String(record[externalId] ?? "");
+          const internalId = result.id;
+          if (externalIdValue && internalId) {
+            insertedRecords.get(objectName).set(externalIdValue, String(internalId));
+          }
+        } catch (err) {
+          errored++;
+          this.logger.warn(`[SeedLoader] Failed to write ${objectName} record`, {
+            error: err.message,
+            recordIndex: i
+          });
+        }
+      } else {
+        const externalIdValue = String(record[externalId] ?? "");
+        if (externalIdValue) {
+          insertedRecords.get(objectName).set(externalIdValue, `dry-run-id-${i}`);
+        }
+        inserted++;
+      }
+    }
+    return {
+      object: objectName,
+      mode,
+      inserted,
+      updated,
+      skipped,
+      errored,
+      total: dataset.records.length,
+      referencesResolved,
+      referencesDeferred,
+      errors
+    };
+  }
+  // ==========================================================================
+  // Internal: Reference Resolution
+  // ==========================================================================
+  async resolveFromDatabase(targetObject, targetField, value) {
+    try {
+      const records = await this.engine.find(targetObject, {
+        filter: { [targetField]: value },
+        select: ["id"],
+        limit: 1
+      });
+      if (records && records.length > 0) {
+        return String(records[0].id || records[0]._id);
+      }
+    } catch {
+    }
+    return null;
+  }
+  async resolveDeferredUpdates(deferredUpdates, insertedRecords, allResults, allErrors) {
+    for (const deferred of deferredUpdates) {
+      const targetMap = insertedRecords.get(deferred.targetObject);
+      let resolvedId = targetMap?.get(String(deferred.attemptedValue));
+      if (!resolvedId) {
+        resolvedId = await this.resolveFromDatabase(
+          deferred.targetObject,
+          deferred.targetField,
+          deferred.attemptedValue
+        ) ?? void 0;
+      }
+      if (resolvedId) {
+        const objectRecordMap = insertedRecords.get(deferred.objectName);
+        const recordId = objectRecordMap?.get(deferred.recordExternalId);
+        if (recordId) {
+          try {
+            await this.engine.update(deferred.objectName, {
+              id: recordId,
+              [deferred.field]: resolvedId
+            });
+            const resultEntry = allResults.find((r) => r.object === deferred.objectName);
+            if (resultEntry) {
+              resultEntry.referencesResolved++;
+              resultEntry.referencesDeferred--;
+            }
+          } catch (err) {
+            this.logger.warn("[SeedLoader] Failed to resolve deferred reference", {
+              object: deferred.objectName,
+              field: deferred.field,
+              error: err.message
+            });
+          }
+        }
+      } else {
+        const error = {
+          sourceObject: deferred.objectName,
+          field: deferred.field,
+          targetObject: deferred.targetObject,
+          targetField: deferred.targetField,
+          attemptedValue: deferred.attemptedValue,
+          recordIndex: deferred.recordIndex,
+          message: `Deferred reference unresolved after pass 2: ${deferred.objectName}.${deferred.field} = '${deferred.attemptedValue}' \u2192 ${deferred.targetObject}.${deferred.targetField} not found`
+        };
+        const resultEntry = allResults.find((r) => r.object === deferred.objectName);
+        if (resultEntry) {
+          resultEntry.errors.push(error);
+        }
+        allErrors.push(error);
+      }
+    }
+  }
+  // ==========================================================================
+  // Internal: Write Operations
+  // ==========================================================================
+  async writeRecord(objectName, record, mode, externalId, existingRecords) {
+    const externalIdValue = record[externalId];
+    const existing = existingRecords?.get(String(externalIdValue ?? ""));
+    switch (mode) {
+      case "insert": {
+        const result = await this.engine.insert(objectName, record);
+        return { action: "inserted", id: this.extractId(result) };
+      }
+      case "update": {
+        if (!existing) {
+          return { action: "skipped" };
+        }
+        const id = this.extractId(existing);
+        await this.engine.update(objectName, { ...record, id });
+        return { action: "updated", id };
+      }
+      case "upsert": {
+        if (existing) {
+          const id = this.extractId(existing);
+          await this.engine.update(objectName, { ...record, id });
+          return { action: "updated", id };
+        } else {
+          const result = await this.engine.insert(objectName, record);
+          return { action: "inserted", id: this.extractId(result) };
+        }
+      }
+      case "ignore": {
+        if (existing) {
+          return { action: "skipped", id: this.extractId(existing) };
+        }
+        const result = await this.engine.insert(objectName, record);
+        return { action: "inserted", id: this.extractId(result) };
+      }
+      case "replace": {
+        const result = await this.engine.insert(objectName, record);
+        return { action: "inserted", id: this.extractId(result) };
+      }
+      default: {
+        const result = await this.engine.insert(objectName, record);
+        return { action: "inserted", id: this.extractId(result) };
+      }
+    }
+  }
+  // ==========================================================================
+  // Internal: Dependency Graph
+  // ==========================================================================
+  /**
+   * Kahn's algorithm for topological sort with cycle detection.
+   */
+  topologicalSort(nodes) {
+    const inDegree = /* @__PURE__ */ new Map();
+    const adjacency = /* @__PURE__ */ new Map();
+    const objectSet = new Set(nodes.map((n) => n.object));
+    for (const node of nodes) {
+      inDegree.set(node.object, 0);
+      adjacency.set(node.object, []);
+    }
+    for (const node of nodes) {
+      for (const dep of node.dependsOn) {
+        if (objectSet.has(dep) && dep !== node.object) {
+          adjacency.get(dep).push(node.object);
+          inDegree.set(node.object, (inDegree.get(node.object) || 0) + 1);
+        }
+      }
+    }
+    const queue = [];
+    for (const [obj, degree] of inDegree) {
+      if (degree === 0) queue.push(obj);
+    }
+    const insertOrder = [];
+    while (queue.length > 0) {
+      const current = queue.shift();
+      insertOrder.push(current);
+      for (const neighbor of adjacency.get(current) || []) {
+        const newDegree = (inDegree.get(neighbor) || 0) - 1;
+        inDegree.set(neighbor, newDegree);
+        if (newDegree === 0) {
+          queue.push(neighbor);
+        }
+      }
+    }
+    const circularDependencies = [];
+    const remaining = nodes.filter((n) => !insertOrder.includes(n.object));
+    if (remaining.length > 0) {
+      const cycles = this.findCycles(remaining);
+      circularDependencies.push(...cycles);
+      for (const node of remaining) {
+        if (!insertOrder.includes(node.object)) {
+          insertOrder.push(node.object);
+        }
+      }
+    }
+    return { insertOrder, circularDependencies };
+  }
+  findCycles(nodes) {
+    const cycles = [];
+    const nodeMap = new Map(nodes.map((n) => [n.object, n]));
+    const visited = /* @__PURE__ */ new Set();
+    const inStack = /* @__PURE__ */ new Set();
+    const dfs = (current, path) => {
+      if (inStack.has(current)) {
+        const cycleStart = path.indexOf(current);
+        if (cycleStart !== -1) {
+          cycles.push([...path.slice(cycleStart), current]);
+        }
+        return;
+      }
+      if (visited.has(current)) return;
+      visited.add(current);
+      inStack.add(current);
+      path.push(current);
+      const node = nodeMap.get(current);
+      if (node) {
+        for (const dep of node.dependsOn) {
+          if (nodeMap.has(dep)) {
+            dfs(dep, [...path]);
+          }
+        }
+      }
+      inStack.delete(current);
+    };
+    for (const node of nodes) {
+      if (!visited.has(node.object)) {
+        dfs(node.object, []);
+      }
+    }
+    return cycles;
+  }
+  // ==========================================================================
+  // Internal: Helpers
+  // ==========================================================================
+  filterByEnv(datasets, env) {
+    if (!env) return datasets;
+    return datasets.filter((d) => d.env.includes(env));
+  }
+  orderDatasets(datasets, insertOrder) {
+    const orderMap = new Map(insertOrder.map((name, i) => [name, i]));
+    return [...datasets].sort((a, b) => {
+      const orderA = orderMap.get(a.object) ?? Number.MAX_SAFE_INTEGER;
+      const orderB = orderMap.get(b.object) ?? Number.MAX_SAFE_INTEGER;
+      return orderA - orderB;
+    });
+  }
+  buildReferenceMap(graph) {
+    const map = /* @__PURE__ */ new Map();
+    for (const node of graph.nodes) {
+      if (node.references.length > 0) {
+        map.set(node.object, node.references);
+      }
+    }
+    return map;
+  }
+  async loadExistingRecords(objectName, externalId) {
+    const map = /* @__PURE__ */ new Map();
+    try {
+      const records = await this.engine.find(objectName, {
+        select: ["id", externalId]
+      });
+      for (const record of records || []) {
+        const key = String(record[externalId] ?? "");
+        if (key) {
+          map.set(key, record);
+        }
+      }
+    } catch {
+    }
+    return map;
+  }
+  looksLikeInternalId(value) {
+    if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) {
+      return true;
+    }
+    if (/^[0-9a-f]{24}$/i.test(value)) {
+      return true;
+    }
+    return false;
+  }
+  extractId(record) {
+    if (!record) return void 0;
+    return String(record.id || record._id || "");
+  }
+  buildEmptyResult(config, durationMs) {
+    return {
+      success: true,
+      dryRun: config.dryRun,
+      dependencyGraph: { nodes: [], insertOrder: [], circularDependencies: [] },
+      results: [],
+      errors: [],
+      summary: {
+        objectsProcessed: 0,
+        totalRecords: 0,
+        totalInserted: 0,
+        totalUpdated: 0,
+        totalSkipped: 0,
+        totalErrored: 0,
+        totalReferencesResolved: 0,
+        totalReferencesDeferred: 0,
+        circularDependencyCount: 0,
+        durationMs
+      }
+    };
+  }
+  buildResult(config, graph, results, errors, durationMs) {
+    const summary = {
+      objectsProcessed: results.length,
+      totalRecords: results.reduce((sum, r) => sum + r.total, 0),
+      totalInserted: results.reduce((sum, r) => sum + r.inserted, 0),
+      totalUpdated: results.reduce((sum, r) => sum + r.updated, 0),
+      totalSkipped: results.reduce((sum, r) => sum + r.skipped, 0),
+      totalErrored: results.reduce((sum, r) => sum + r.errored, 0),
+      totalReferencesResolved: results.reduce((sum, r) => sum + r.referencesResolved, 0),
+      totalReferencesDeferred: results.reduce((sum, r) => sum + r.referencesDeferred, 0),
+      circularDependencyCount: graph.circularDependencies.length,
+      durationMs
+    };
+    const hasErrors = errors.length > 0 || summary.totalErrored > 0;
+    return {
+      success: !hasErrors,
+      dryRun: config.dryRun,
+      dependencyGraph: graph,
+      results,
+      errors,
+      summary
+    };
+  }
+};
+
 // src/app-plugin.ts
 var AppPlugin = class {
   constructor(bundle) {
@@ -144,20 +656,52 @@ var AppPlugin = class {
       };
       if (seedDatasets.length > 0) {
         ctx.logger.info(`[AppPlugin] Found ${seedDatasets.length} seed datasets for ${appId}`);
-        for (const dataset of seedDatasets) {
-          if (dataset.object && Array.isArray(dataset.records)) {
-            const objectFQN = toFQN(dataset.object);
-            ctx.logger.info(`[Seeder] Seeding ${dataset.records.length} records for ${objectFQN}`);
+        const normalizedDatasets = seedDatasets.filter((d) => d.object && Array.isArray(d.records)).map((d) => ({
+          ...d,
+          object: toFQN(d.object)
+        }));
+        try {
+          const metadata = ctx.getService("metadata");
+          if (metadata) {
+            const seedLoader = new SeedLoaderService(ql, metadata, ctx.logger);
+            const { SeedLoaderRequestSchema } = await import("@objectstack/spec/data");
+            const request = SeedLoaderRequestSchema.parse({
+              datasets: normalizedDatasets,
+              config: { defaultMode: "upsert", multiPass: true }
+            });
+            const result = await seedLoader.load(request);
+            ctx.logger.info("[Seeder] Seed loading complete", {
+              inserted: result.summary.totalInserted,
+              updated: result.summary.totalUpdated,
+              errors: result.errors.length
+            });
+          } else {
+            ctx.logger.debug("[Seeder] No metadata service; using basic insert fallback");
+            for (const dataset of normalizedDatasets) {
+              ctx.logger.info(`[Seeder] Seeding ${dataset.records.length} records for ${dataset.object}`);
+              for (const record of dataset.records) {
+                try {
+                  await ql.insert(dataset.object, record);
+                } catch (err) {
+                  ctx.logger.warn(`[Seeder] Failed to insert ${dataset.object} record:`, { error: err.message });
+                }
+              }
+            }
+            ctx.logger.info("[Seeder] Data seeding complete.");
+          }
+        } catch (err) {
+          ctx.logger.warn("[Seeder] SeedLoaderService failed, falling back to basic insert", { error: err.message });
+          for (const dataset of normalizedDatasets) {
             for (const record of dataset.records) {
               try {
-                await ql.insert(objectFQN, record);
-              } catch (err) {
-                ctx.logger.warn(`[Seeder] Failed to insert ${objectFQN} record:`, { error: err.message });
+                await ql.insert(dataset.object, record);
+              } catch (insertErr) {
+                ctx.logger.warn(`[Seeder] Failed to insert ${dataset.object} record:`, { error: insertErr.message });
               }
             }
           }
+          ctx.logger.info("[Seeder] Data seeding complete (fallback).");
         }
-        ctx.logger.info("[Seeder] Data seeding complete.");
       }
     };
     this.bundle = bundle;
@@ -1616,6 +2160,7 @@ export {
   RouteGroupBuilder,
   RouteManager,
   Runtime,
+  SeedLoaderService,
   createDispatcherPlugin,
   createRestApiPlugin
 };