@tenonhq/dovetail-core 0.0.105 → 0.0.106

package/dist/commander.js

@@ -388,7 +388,7 @@ async function initCommands() {
             await (0, schemaCommand_1.schemaSnapshotsCommand)(args);
         }
     })
-        .command("reconcile", "Make your personal instance match the checked-out branch (diff/report by default; --apply to apply UPDATE + DELETE)", (cmdArgs) => {
+        .command("reconcile", "Make your personal instance match the checked-out branch (diff/report by default; --apply to apply CREATE + UPDATE + DELETE)", (cmdArgs) => {
         cmdArgs.options({
             ...sharedOptions,
             scope: {
@@ -409,7 +409,7 @@ async function initCommands() {
             apply: {
                 type: "boolean",
                 default: false,
-                describe: "Apply the safe subset: record UPDATE (branch -> instance) + tracked DELETE. Refuses on drift unless --force. CREATE is Phase 3.",
+                describe: "Apply records branch -> instance: CREATE + UPDATE + tracked DELETE. Refuses on drift unless --force.",
             },
             force: {
                 type: "boolean",

package/dist/reconcile/applyPlan.js

@@ -12,13 +12,14 @@
 //                              since baseline; refresh to keep them or --force.
 //   - Clean, or --force      -> apply.
 //
-// What applies, in Phase 2:
+// What applies:
+//   - CREATE  every branch-only record (created on the instance with the
+//             branch's own sys_id — the server honors it via setNewGuidValue).
 //   - UPDATE  every diff update (branch content overwrites the instance).
 //   - DELETE  only "tracked" instance-only records (the branch deleted them).
 //             "local-new" / "no-baseline" records are the dev's own creations —
 //             NEVER deleted, not even with --force (force discards edits to the
 //             branch's records, it does not reap records the branch never owned).
-//   - CREATE  deferred to Phase 3 — reported, never applied here.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildApplyPlan = buildApplyPlan;
 function buildApplyPlan(options) {
@@ -51,9 +52,9 @@ function buildApplyPlan(options) {
     return {
         refuse,
         refuseReason,
+        creates: diff.creates.slice(),
         updates: diff.updates.slice(),
         deletes,
         skippedDeletes,
-        deferredCreates: diff.creates.slice(),
     };
 }

package/dist/reconcile/multiPass.js

@@ -0,0 +1,55 @@
+"use strict";
+// FK-safe record operations without building a dependency graph. ServiceNow
+// refuses to delete a record another still references, and refuses to create a
+// record before the parent it references exists. Both orderings fall out of the
+// same loop: attempt every pending record, keep whatever succeeded, retry the
+// failures next pass, stop when a full pass makes no progress (a genuine,
+// non-ordering failure — e.g. ACL) and report the stragglers.
+//   - DELETE: a parent blocked by a still-present child succeeds once the child
+//     is gone (children before parents).
+//   - CREATE: a child blocked by a missing parent succeeds once the parent
+//     exists (parents before children).
+//
+// Pure control flow: the operation is injected, so the loop is unit-tested with
+// a fake executor that encodes a dependency ("B cannot proceed until A has").
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runMultiPassOps = runMultiPassOps;
+async function runMultiPassOps(changes, attempt) {
+    const succeeded = [];
+    let pending = changes.slice();
+    let lastError = new Map();
+    while (pending.length > 0) {
+        const stillPending = [];
+        let progressed = false;
+        lastError = new Map();
+        for (const change of pending) {
+            let result;
+            try {
+                result = await attempt(change);
+            }
+            catch (e) {
+                result = { ok: false, error: e instanceof Error ? e.message : String(e) };
+            }
+            if (result.ok) {
+                succeeded.push({ change, ok: true });
+                progressed = true;
+            }
+            else {
+                stillPending.push(change);
+                lastError.set(change.sys_id, result.error || "operation failed");
+            }
+        }
+        pending = stillPending;
+        if (!progressed) {
+            // A full pass changed nothing — the remaining failures are not ordering
+            // problems. Report them and stop.
+            break;
+        }
+    }
+    const failures = pending.map((change) => ({
+        change,
+        ok: false,
+        error: lastError.get(change.sys_id) || "operation failed",
+    }));
+    return succeeded.concat(failures);
+}

package/dist/reconcileCommand.js

@@ -9,10 +9,13 @@
 //                      report schema drift (report-only), print the report.
 //   --write-baseline   establish the per-instance merge-base from current live
 //                      state (no record changes). Run this once before --apply.
-//   --apply [--force]  apply the safe subset: record UPDATE (branch -> instance)
-//                      and tracked DELETE. Refuses if the instance drifted since
-//                      baseline (unless --force). CREATE is deferred to Phase 3;
-//                      schema stays report-only (a ServiceNow ceiling).
+//   --apply [--force]  apply records branch -> instance: CREATE net-new (with
+//                      the branch's own sys_id, which the server honors),
+//                      UPDATE changed records, DELETE tracked instance-only
+//                      records. Refuses if the instance drifted since baseline
+//                      (unless --force). Schema stays report-only (a SN ceiling).
+//                      Updates for a scope with no configured update set are
+//                      skipped, not pushed — a raw PATCH would re-parent scope.
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -53,7 +56,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.reconcileCommand = reconcileCommand;
 const path_1 = __importDefault(require("path"));
 const os_1 = __importDefault(require("os"));
-const fs_1 = require("fs");
+const fs_1 = __importStar(require("fs"));
 const child_process_1 = require("child_process");
 const util_1 = require("util");
 const dovetail_schema_1 = require("@tenonhq/dovetail-schema");
@@ -63,13 +66,33 @@ const commands_1 = require("./commands");
 const ConfigManager = __importStar(require("./config"));
 const snClient_1 = require("./snClient");
 const appUtils_1 = require("./appUtils");
+const projectFiles_1 = require("./projectFiles");
 const recordSource_1 = require("./reconcile/recordSource");
 const recordDiff_1 = require("./reconcile/recordDiff");
+const fields_1 = require("./reconcile/fields");
 const baseline_1 = require("./reconcile/baseline");
 const report_1 = require("./reconcile/report");
 const applyPlan_1 = require("./reconcile/applyPlan");
-const deleteOrder_1 = require("./reconcile/deleteOrder");
+const multiPass_1 = require("./reconcile/multiPass");
 const gitignore_1 = require("./reconcile/gitignore");
+// Per-scope update-set routing, read from the same .dove-update-sets.json the
+// push pipeline uses. Missing/parse errors degrade to {} (treated as "no update
+// set configured"), never throw.
+function getUpdateSetConfig() {
+    try {
+        const configPath = (0, projectFiles_1.getUpdateSetsConfigPath)();
+        if (fs_1.default.existsSync(configPath)) {
+            const parsed = JSON.parse(fs_1.default.readFileSync(configPath, "utf8"));
+            if (parsed && typeof parsed === "object") {
+                return parsed;
+            }
+        }
+    }
+    catch (e) {
+        FileLogger_1.fileLogger.debug("reconcile: could not read update-set config");
+    }
+    return {};
+}
 const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
 function requireCreds() {
     const { SN_USER = "", SN_PASSWORD = "", SN_INSTANCE = "" } = process.env;
@@ -218,7 +241,7 @@ async function applyDeletes(scope, deletes) {
         return [];
     }
     const client = (0, snClient_1.defaultClient)();
-    return (0, deleteOrder_1.runMultiPassDeletes)(deletes, async (change) => {
+    return (0, multiPass_1.runMultiPassOps)(deletes, async (change) => {
         try {
             const response = await client.deleteRecord({
                 table: change.table,
@@ -239,6 +262,117 @@ async function applyDeletes(scope, deletes) {
         }
     });
 }
+// Reconstruct the full field set for a CREATE from the record's on-disk
+// representation: the metaData.json snapshot supplies every captured field
+// value, then the canonical field files (script.js, etc.) override their own
+// fields. Volatile bookkeeping and identity columns are excluded — sys_id is
+// passed separately (and honored by the server), sys_scope comes from `scope`.
+async function buildCreateFields(scope, change) {
+    const sourcePath = ConfigManager.getSourcePathForScope(scope);
+    const recordDir = path_1.default.join(sourcePath, change.table, change.name);
+    const fields = {};
+    // 1. metaData snapshot — short/non-file fields (active, order, name, ...).
+    try {
+        const metaRaw = await fs_1.promises.readFile(path_1.default.join(recordDir, "metaData.json"), "utf8");
+        const meta = JSON.parse(metaRaw);
+        for (const key of Object.keys(meta)) {
+            if (key.charAt(0) === "_" || key === "sys_id" || key === "sys_scope") {
+                continue;
+            }
+            if (key.indexOf("sys_") === 0) {
+                // sys_created_on/by, sys_updated_on/by, sys_mod_count, sys_class_name, ...
+                // Skipped: server-managed. (sys_class_name on a base table defaults fine.)
+                continue;
+            }
+            const value = meta[key];
+            if (typeof value === "string") {
+                fields[key] = value;
+            }
+            else if (typeof value === "object" &&
+                value !== null &&
+                typeof value.value === "string") {
+                fields[key] = value.value;
+            }
+        }
+    }
+    catch (e) {
+        FileLogger_1.fileLogger.debug("reconcile: no metaData for create " + recordDir);
+    }
+    // 2. Canonical field files override (the authoritative large-field content).
+    let entries = [];
+    try {
+        entries = await fs_1.promises.readdir(recordDir);
+    }
+    catch (e) {
+        FileLogger_1.fileLogger.debug("reconcile: create record dir missing " + recordDir);
+    }
+    for (const entry of entries) {
+        if (entry === "metaData.json") {
+            continue;
+        }
+        const dot = entry.lastIndexOf(".");
+        if (dot <= 0) {
+            continue;
+        }
+        const name = entry.slice(0, dot);
+        const type = entry.slice(dot + 1);
+        if (!(0, fields_1.isComparableField)({ name, type })) {
+            continue;
+        }
+        try {
+            const stat = await fs_1.promises.stat(path_1.default.join(recordDir, entry));
+            if (stat.isFile()) {
+                fields[name] = await fs_1.promises.readFile(path_1.default.join(recordDir, entry), "utf8");
+            }
+        }
+        catch (e) {
+            // ignore unreadable entry
+        }
+    }
+    // Ensure a name is present (some tables require it).
+    if (!fields.name && change.name) {
+        fields.name = change.name;
+    }
+    return fields;
+}
+async function applyCreates(scope, creates, updateSetSysId) {
+    if (creates.length === 0) {
+        return [];
+    }
+    const client = (0, snClient_1.defaultClient)();
+    return (0, multiPass_1.runMultiPassOps)(creates, async (change) => {
+        try {
+            const fields = await buildCreateFields(scope, change);
+            const response = await client.createRecord({
+                table: change.table,
+                fields,
+                sys_id: change.sys_id,
+                scope,
+                update_set_sys_id: updateSetSysId,
+            });
+            let data = response.data;
+            if (data && data.result) {
+                data = data.result;
+            }
+            if (data && data.error) {
+                return { ok: false, error: data.error };
+            }
+            if (data && data.sys_id && data.sys_id !== change.sys_id) {
+                // The server reassigned the sys_id — identity would diverge and the
+                // diff would never converge. Surface it loudly rather than silently
+                // creating a phantom.
+                return {
+                    ok: false,
+                    error: "server reassigned sys_id (" + data.sys_id + " != " + change.sys_id + ")",
+                };
+            }
+            return { ok: true };
+        }
+        catch (e) {
+            return { ok: false, error: e instanceof Error ? e.message : String(e) };
+        }
+    });
+}
 // Capture current live state across every scope into a fresh baseline + ensure
 // the baseline file is gitignored in the consumer project.
 async function recaptureBaseline(options) {
@@ -364,23 +498,45 @@ async function applyMode(options) {
     }
     Logger_1.logger.info("");
     Logger_1.logger.info("Applying reconcile (branch -> instance)...");
+    const updateSetConfig = getUpdateSetConfig();
+    let totalCreated = 0;
     let totalUpdated = 0;
     let totalDeleted = 0;
     const failures = [];
     for (const entry of plans) {
         const { scope, plan } = entry;
-        if (plan.deferredCreates.length > 0) {
-            Logger_1.logger.info("  [" +
+        const updateSet = updateSetConfig[scope];
+        const updateSetSysId = updateSet ? updateSet.sys_id : undefined;
+        // CREATE — scope-correct via the `scope` param; routed into the update set
+        // when one is configured.
+        const createOutcomes = await applyCreates(scope, plan.creates, updateSetSysId);
+        for (const outcome of createOutcomes) {
+            if (outcome.ok) {
+                totalCreated++;
+            }
+            else {
+                failures.push("[" + scope + "] create " + outcome.change.table + "/" + outcome.change.name + ": " + (outcome.error || "failed"));
+            }
+        }
+        // UPDATE — guard against the re-parenting trap: pushFiles falls back to a
+        // raw PATCH (which adopts session scope) when no update set is configured.
+        // Refuse to push updates for such a scope rather than silently re-parent.
+        if (plan.updates.length > 0 && !updateSetSysId) {
+            failures.push("[" +
                 scope +
                 "] " +
-                plan.deferredCreates.length +
-                " create(s) deferred — CREATE ships in Phase 3.");
+                plan.updates.length +
+                " update(s) skipped — no update set configured for this scope. " +
+                "Set one (e.g. `dove switchUpdateSet`) before applying updates, to avoid re-parenting scope.");
         }
-        const updateResult = await applyUpdates(scope, plan.updates);
-        totalUpdated += updateResult.pushed;
-        for (const failure of updateResult.failures) {
-            failures.push("[" + scope + "] update: " + failure);
+        else {
+            const updateResult = await applyUpdates(scope, plan.updates);
+            totalUpdated += updateResult.pushed;
+            for (const failure of updateResult.failures) {
+                failures.push("[" + scope + "] update: " + failure);
+            }
         }
+        // DELETE — tracked instance-only records.
         const deleteOutcomes = await applyDeletes(scope, plan.deletes);
         for (const outcome of deleteOutcomes) {
             if (outcome.ok) {
@@ -398,7 +554,13 @@ async function applyMode(options) {
         scopes: options.scopes,
     });
     Logger_1.logger.info("");
-    Logger_1.logger.success("Applied: " + totalUpdated + " updated, " + totalDeleted + " deleted.");
+    Logger_1.logger.success("Applied: " +
+        totalCreated +
+        " created, " +
+        totalUpdated +
+        " updated, " +
+        totalDeleted +
+        " deleted.");
     if (failures.length > 0) {
         Logger_1.logger.error(failures.length + " operation(s) failed:");
         for (const failure of failures) {

package/dist/tests/reconcileApplyPlan.test.js

@@ -54,7 +54,7 @@ describe("buildApplyPlan — refusal gating", () => {
     });
 });
 describe("buildApplyPlan — what applies", () => {
-    it("includes updates and defers creates", () => {
+    it("includes updates and creates", () => {
         const plan = (0, applyPlan_1.buildApplyPlan)({
             diff: diff({
                 updates: [change({ sys_id: "u", kind: "update" })],
@@ -65,7 +65,7 @@ describe("buildApplyPlan — what applies", () => {
             force: false,
         });
         expect(plan.updates.map((u) => u.sys_id)).toEqual(["u"]);
-        expect(plan.deferredCreates.map((c) => c.sys_id)).toEqual(["c"]);
+        expect(plan.creates.map((c) => c.sys_id)).toEqual(["c"]);
     });
     it("applies only tracked deletes; keeps local-new and no-baseline", () => {
         const plan = (0, applyPlan_1.buildApplyPlan)({

package/dist/tests/reconcileMultiPass.test.js

@@ -0,0 +1,74 @@
+"use strict";
+// Tests for the generic multi-pass op loop: clean ops, emergent ordering (a
+// record blocked until its dependency resolves), a genuine failure surviving a
+// no-progress pass, and thrown-error handling. Exercised for both delete
+// ordering (children before parents) and create ordering (parents before
+// children).
+Object.defineProperty(exports, "__esModule", { value: true });
+const multiPass_1 = require("../reconcile/multiPass");
+const rec = (sys_id, kind) => ({
+    kind,
+    table: "sys_script",
+    scope: "x_cadso_core",
+    sys_id,
+    name: sys_id,
+    fieldDeltas: [],
+});
+describe("runMultiPassOps", () => {
+    it("completes everything in a single clean pass", async () => {
+        const calls = [];
+        const exec = async (c) => {
+            calls.push(c.sys_id);
+            return { ok: true };
+        };
+        const out = await (0, multiPass_1.runMultiPassOps)([rec("a", "delete"), rec("b", "delete")], exec);
+        expect(out.every((o) => o.ok)).toBe(true);
+        expect(calls.sort()).toEqual(["a", "b"]);
+    });
+    it("resolves DELETE ordering: parent succeeds only after child is gone", async () => {
+        const deleted = new Set();
+        const exec = async (c) => {
+            if (c.sys_id === "parent" && !deleted.has("child")) {
+                return { ok: false, error: "still referenced by child" };
+            }
+            deleted.add(c.sys_id);
+            return { ok: true };
+        };
+        const out = await (0, multiPass_1.runMultiPassOps)([rec("parent", "delete"), rec("child", "delete")], exec);
+        expect(out.every((o) => o.ok)).toBe(true);
+    });
+    it("resolves CREATE ordering: child succeeds only after parent exists", async () => {
+        const created = new Set();
+        const exec = async (c) => {
+            if (c.sys_id === "child" && !created.has("parent")) {
+                return { ok: false, error: "parent does not exist yet" };
+            }
+            created.add(c.sys_id);
+            return { ok: true };
+        };
+        // child attempted first, fails, then succeeds on the pass after parent.
+        const out = await (0, multiPass_1.runMultiPassOps)([rec("child", "create"), rec("parent", "create")], exec);
+        expect(out.every((o) => o.ok)).toBe(true);
+        expect(created.has("child")).toBe(true);
+    });
+    it("reports a record that fails for a non-ordering reason", async () => {
+        const exec = async (c) => c.sys_id === "locked" ? { ok: false, error: "ACL blocked" } : { ok: true };
+        const out = await (0, multiPass_1.runMultiPassOps)([rec("ok", "delete"), rec("locked", "delete")], exec);
+        const failed = out.filter((o) => !o.ok);
+        expect(failed).toHaveLength(1);
+        expect(failed[0].change.sys_id).toBe("locked");
+        expect(failed[0].error).toBe("ACL blocked");
+    });
+    it("treats a thrown executor error as a failure", async () => {
+        const exec = async () => {
+            throw new Error("network down");
+        };
+        const out = await (0, multiPass_1.runMultiPassOps)([rec("a", "create")], exec);
+        expect(out[0].ok).toBe(false);
+        expect(out[0].error).toBe("network down");
+    });
+    it("returns [] for no changes", async () => {
+        const out = await (0, multiPass_1.runMultiPassOps)([], async () => ({ ok: true }));
+        expect(out).toEqual([]);
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tenonhq/dovetail-core",
-  "version": "0.0.105",
+  "version": "0.0.106",
   "description": "Next-gen file syncer",
   "license": "GPL-3.0",
   "main": "./dist/index.js",

package/dist/reconcile/deleteOrder.js

@@ -1,52 +0,0 @@
-"use strict";
-// FK-safe deletion without building a dependency graph. ServiceNow refuses to
-// delete a record another record still references; the safe order is therefore
-// children before parents. Rather than model the foreign-key graph, delete in
-// passes: attempt every pending record, keep whatever succeeded, and retry the
-// failures on the next pass. A record blocked by a still-present dependent
-// succeeds once that dependent is gone. Stop when a full pass makes no progress
-// (a genuine, non-ordering failure — e.g. ACL) and report the stragglers.
-//
-// Pure control flow: the actual delete is injected, so the loop is unit-tested
-// with a fake executor that encodes "B cannot delete until A is gone."
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runMultiPassDeletes = runMultiPassDeletes;
-async function runMultiPassDeletes(changes, attemptDelete) {
-    const succeeded = [];
-    let pending = changes.slice();
-    let lastError = new Map();
-    while (pending.length > 0) {
-        const stillPending = [];
-        let progressed = false;
-        lastError = new Map();
-        for (const change of pending) {
-            let result;
-            try {
-                result = await attemptDelete(change);
-            }
-            catch (e) {
-                result = { ok: false, error: e instanceof Error ? e.message : String(e) };
-            }
-            if (result.ok) {
-                succeeded.push({ change, ok: true });
-                progressed = true;
-            }
-            else {
-                stillPending.push(change);
-                lastError.set(change.sys_id, result.error || "delete failed");
-            }
-        }
-        pending = stillPending;
-        if (!progressed) {
-            // A full pass deleted nothing — the remaining failures are not ordering
-            // problems. Report them and stop.
-            break;
-        }
-    }
-    const failures = pending.map((change) => ({
-        change,
-        ok: false,
-        error: lastError.get(change.sys_id) || "delete failed",
-    }));
-    return succeeded.concat(failures);
-}

package/dist/tests/reconcileDeleteOrder.test.js

@@ -1,70 +0,0 @@
-"use strict";
-// Tests for the multi-pass delete loop: clean deletes, emergent FK ordering
-// (a record blocked until its dependent is gone), and a genuine failure that
-// survives a no-progress pass and is reported.
-Object.defineProperty(exports, "__esModule", { value: true });
-const deleteOrder_1 = require("../reconcile/deleteOrder");
-const del = (sys_id) => ({
-    kind: "delete",
-    table: "sys_script",
-    scope: "x_cadso_core",
-    sys_id,
-    name: sys_id,
-    fieldDeltas: [],
-    deleteDisposition: "tracked",
-});
-describe("runMultiPassDeletes", () => {
-    it("deletes everything in a single clean pass", async () => {
-        const calls = [];
-        const exec = async (c) => {
-            calls.push(c.sys_id);
-            return { ok: true };
-        };
-        const out = await (0, deleteOrder_1.runMultiPassDeletes)([del("a"), del("b")], exec);
-        expect(out.every((o) => o.ok)).toBe(true);
-        expect(calls.sort()).toEqual(["a", "b"]);
-    });
-    it("resolves FK ordering: parent succeeds only after the child is gone", async () => {
-        // "parent" cannot delete while "child" still exists.
-        const deleted = new Set();
-        const exec = async (c) => {
-            if (c.sys_id === "parent" && !deleted.has("child")) {
-                return { ok: false, error: "still referenced by child" };
-            }
-            deleted.add(c.sys_id);
-            return { ok: true };
-        };
-        // parent is attempted first, fails, then succeeds on the pass after child.
-        const out = await (0, deleteOrder_1.runMultiPassDeletes)([del("parent"), del("child")], exec);
-        expect(out.filter((o) => o.ok).map((o) => o.change.sys_id).sort()).toEqual([
-            "child",
-            "parent",
-        ]);
-        expect(out.every((o) => o.ok)).toBe(true);
-    });
-    it("reports a record that fails for a non-ordering reason", async () => {
-        const exec = async (c) => {
-            if (c.sys_id === "locked") {
-                return { ok: false, error: "ACL blocked" };
-            }
-            return { ok: true };
-        };
-        const out = await (0, deleteOrder_1.runMultiPassDeletes)([del("ok"), del("locked")], exec);
-        const failed = out.filter((o) => !o.ok);
-        expect(failed).toHaveLength(1);
-        expect(failed[0].change.sys_id).toBe("locked");
-        expect(failed[0].error).toBe("ACL blocked");
-    });
-    it("treats a thrown executor error as a failure", async () => {
-        const exec = async () => {
-            throw new Error("network down");
-        };
-        const out = await (0, deleteOrder_1.runMultiPassDeletes)([del("a")], exec);
-        expect(out[0].ok).toBe(false);
-        expect(out[0].error).toBe("network down");
-    });
-    it("returns [] for no changes", async () => {
-        const out = await (0, deleteOrder_1.runMultiPassDeletes)([], async () => ({ ok: true }));
-        expect(out).toEqual([]);
-    });
-});