@tenonhq/dovetail-core 0.0.85 → 0.0.87

package/README.md

@@ -1,6 +1,55 @@
 # @tenonhq/dovetail-core
 
-This module contains the core of Dovetail. It is required to use Dovetail at all.
-It can interact with other plugins after you configure them.
+The core of Dovetail — ships the `dove` CLI binary and the sync engine that connects local source files to a ServiceNow instance via REST.
 
+## What's in this package
 
+- **`dove` CLI** — command router built on yargs. Watch, push, pull, build, deploy, scope/update-set management, record CRUD, ClickUp integration, schema pull, dashboard launch, and the project migration command (`dove migrate`).
+- **Sync engine** — file watcher, manifest-driven file ↔ ServiceNow record mapping, plugin pipeline runner, Scripted REST API client.
+- **Skills** — `skills/` ships ~8 Claude Code skill markdowns. `dove init-claude` copies them into `<cwd>/.claude/commands/` for in-project Claude usage.
+
+## Installation
+
+```bash
+nvm use 20
+npm i -D @tenonhq/dovetail-core
+npx dove init        # scaffolds dove.config.js
+npx dove configure   # creates .env (do not commit)
+```
+
+## Commands
+
+The full command surface is documented in the [root CLAUDE.md](../../CLAUDE.md#essential-commands). Source of truth lives in [`src/commander.ts`](src/commander.ts).
+
+Quick reference:
+
+```bash
+npx dove watch         # multi-scope watch + dashboard
+npx dove push          # build + push current files
+npx dove refresh       # pull manifest + new files
+npx dove build         # local build only
+npx dove deploy        # deploy built artifacts
+npx dove dashboard     # update-set dashboard web UI
+npx dove migrate       # migrate a Sincronia project to Dovetail (dry-run by default; --apply to write)
+```
+
+See [`UPDATE_SET_COMMANDS.md`](UPDATE_SET_COMMANDS.md) for the full update-set CLI surface.
+
+## Plugins
+
+Dovetail's build pipeline is plugin-driven. Each plugin is an npm package implementing `run(context, content, options) => Promise<PluginResults>`. Shipped plugins live in sibling packages (`@tenonhq/dovetail-typescript-plugin`, `-babel-plugin`, `-webpack-plugin`, `-sass-plugin`, `-eslint-plugin`, `-prettier-plugin`, `-babel-preset-servicenow`, `-babel-plugin-remove-modules`).
+
+## Related packages
+
+| Package | Purpose |
+|---|---|
+| `@tenonhq/dovetail-types` | Shared `Sinc.*` + `SN.*` type namespaces |
+| `@tenonhq/dovetail-schema` | ServiceNow table schema fetcher |
+| `@tenonhq/dovetail-dashboard` | Update-set dashboard web UI |
+| `@tenonhq/dovetail-servicenow` | Platform helpers + `dove-sn` build-flow CLI |
+| `@tenonhq/dovetail-sawmill` | Cross-instance update-set retrieve/preview/commit |
+| `@tenonhq/dovetail-mcp` | Read-only MCP server (ClickUp, Gmail, Calendar, ServiceNow) |
+
+## Full spec
+
+See [`docs/dovetail-platform-spec.md`](../../docs/dovetail-platform-spec.md) for the complete architecture, type system, REST API contract, build pipeline, and sync mechanism.

package/dist/FileLogger.js

@@ -41,10 +41,15 @@ exports.enableFileLogging = enableFileLogging;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const chalk_1 = __importDefault(require("chalk"));
+// Roll over to a fresh log file once the current one has had this many writes.
+// Long-running sessions (e.g. `dove watch`) would otherwise grow a single
+// debug log unbounded; capping at 200 writes per file keeps each one small.
+const ROTATION_WRITE_LIMIT = 200;
 class FileLogger {
     logFilePath;
     logStream = null;
     initialized = false;
+    writeCount = 0;
     constructor() {
         // Initialize on first use
         this.logFilePath = "";
@@ -63,7 +68,8 @@ class FileLogger {
             // Create or append to the log file
             this.logStream = fs.createWriteStream(this.logFilePath, { flags: 'a' });
             this.initialized = true;
-            // Write header to log file
+            this.writeCount = 0;
+            // Write header to log file — once, at the top of each (rotated) file
             this.writeToFile(`\n${"=".repeat(80)}`);
             this.writeToFile(`Dovetail Debug Log - Started at ${new Date().toISOString()}`);
             this.writeToFile(`Log file: ${this.logFilePath}`);
@@ -76,6 +82,29 @@ class FileLogger {
             console.error(chalk_1.default.red(`Failed to create log file: ${error}`));
         }
     }
+    /**
+     * Close the current log file and start a new one once it has reached
+     * ROTATION_WRITE_LIMIT writes. Keeps any single debug log from growing huge
+     * during long watch sessions.
+     */
+    rotateIfNeeded() {
+        if (!this.initialized)
+            return;
+        if (this.writeCount < ROTATION_WRITE_LIMIT)
+            return;
+        if (this.logStream && this.logStream.writable) {
+            const timestamp = new Date().toISOString();
+            this.logStream.write(`[${timestamp}] ${"=".repeat(80)}\n`);
+            this.logStream.write(`[${timestamp}] Log rotated after ${ROTATION_WRITE_LIMIT} writes — continues in a new file\n`);
+            this.logStream.write(`[${timestamp}] ${"=".repeat(80)}\n`);
+        }
+        if (this.logStream) {
+            this.logStream.end();
+            this.logStream = null;
+        }
+        this.initialized = false;
+        this.initialize();
+    }
     /**
      * Write a message to the log file
      */
@@ -83,9 +112,13 @@ class FileLogger {
         if (!this.initialized) {
             this.initialize();
         }
+        else {
+            this.rotateIfNeeded();
+        }
         if (this.logStream && this.logStream.writable) {
             const timestamp = new Date().toISOString();
             this.logStream.write(`[${timestamp}] ${message}\n`);
+            this.writeCount++;
         }
     }
     /**

package/dist/flowDesigner/values.js

@@ -45,11 +45,28 @@ function decodeV2Values(blob) {
 /**
  * Encode a V2 values payload back to ServiceNow's storage format.
  *
- * Inverse of decodeV2Values. Provided for symmetry, diagnostic round-trip
- * tests, and any future write path. Current Craftsman sync treats sys_hub_*
- * as _readOnlyTables, so this is not invoked on the push path today.
+ * Inverse of decodeV2Values. The output's gzip envelope may differ byte-for-byte
+ * from a Java-produced original (Deflater and zlib emit different but compatible
+ * streams), but the decompressed JSON is identical content. Round-trip
+ * fidelity is verified against real Tenon-shipped fixtures in
+ * tests/v2Values.test.ts; SN-side acceptance is verified by the
+ * `@tenonhq/sincronia-servicenow` flowDesigner write path.
+ *
+ * Throws when `value` is undefined or otherwise not JSON-serializable
+ * (e.g. functions, BigInt). Anything that survives JSON.stringify will be
+ * accepted; structural validity is the caller's responsibility.
  */
-function encodeV2Values(value) {
+function encodeV2Values(value, opts = {}) {
+    if (value === undefined) {
+        throw new Error("encodeV2Values: value must not be undefined");
+    }
     const json = JSON.stringify(value);
-    return (0, zlib_1.gzipSync)(Buffer.from(json, "utf8")).toString("base64");
+    if (json === undefined) {
+        throw new Error("encodeV2Values: value is not JSON-serializable (e.g. function or BigInt at the root)");
+    }
+    const zlibOpts = {};
+    if (opts.level !== undefined && opts.level !== null) {
+        zlibOpts.level = opts.level;
+    }
+    return (0, zlib_1.gzipSync)(Buffer.from(json, "utf8"), zlibOpts).toString("base64");
 }

package/dist/logMessages.js

@@ -7,6 +7,7 @@ exports.logBuildResults = exports.logPushResults = exports.log = void 0;
 exports.logFilePush = logFilePush;
 exports.logDeploy = logDeploy;
 const Logger_1 = require("./Logger");
+const FileLogger_1 = require("./FileLogger");
 const chalk_1 = __importDefault(require("chalk"));
 exports.log = console.log;
 function parseError(err) {
@@ -20,14 +21,18 @@ function logFilePush(context, res) {
     const instance = process.env.SN_INSTANCE || "instance";
     const label = chalk_1.default.bold.blue;
     const timestamp = new Date().toLocaleTimeString();
+    const fileLabel = context.tableName + "/" + context.name + " (" + context.targetField + ")";
     if (success) {
-        Logger_1.logger.info(chalk_1.default.green("Pushed") + " " + context.tableName + "/" + context.name +
-            " (" + context.targetField + ") to " + instance + " at " + timestamp);
+        Logger_1.logger.info(chalk_1.default.green("Pushed") + " " + fileLabel + " to " + instance +
+            " at " + timestamp);
+        // Persist every push to the session log file (file-only, no console echo).
+        // The log file rotates every 200 writes (see FileLogger) so it never grows huge.
+        FileLogger_1.fileLogger.debug("Pushed " + fileLabel + " to " + instance + " at " + timestamp);
     }
     else {
-        Logger_1.logger.error("Failed to push " + context.tableName + "/" + context.name +
-            " (" + context.targetField + ") to " + instance);
+        Logger_1.logger.error("Failed to push " + fileLabel + " to " + instance);
         Logger_1.logger.error(message);
+        FileLogger_1.fileLogger.debug("Failed to push " + fileLabel + " to " + instance + ": " + message);
     }
 }
 function multiLog(files, success, resultSet, successMessage, errorMessage, err) {

package/dist/tests/v2Values.test.js

@@ -1,7 +1,43 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
 const zlib_1 = require("zlib");
 const values_1 = require("../flowDesigner/values");
+const FIXTURES_DIR = path.join(__dirname, "fixtures", "v2-blobs");
 describe("decodeV2Values / encodeV2Values", () => {
     describe("round-trip", () => {
         it("decodes what encodeV2Values encoded (primitive array)", () => {
@@ -76,4 +112,81 @@ describe("decodeV2Values / encodeV2Values", () => {
             expect(decoded[0].id).toBe("1");
         });
     });
+    describe("encode — input validation", () => {
+        it("throws on undefined", () => {
+            expect(() => (0, values_1.encodeV2Values)(undefined)).toThrow(/must not be undefined/);
+        });
+        it("throws on non-JSON-serializable values (function at root)", () => {
+            expect(() => (0, values_1.encodeV2Values)(() => 1)).toThrow(/not JSON-serializable/);
+        });
+        it("throws on non-JSON-serializable values (BigInt at root)", () => {
+            expect(() => (0, values_1.encodeV2Values)(BigInt(1))).toThrow();
+        });
+        it("accepts null", () => {
+            expect((0, values_1.decodeV2Values)((0, values_1.encodeV2Values)(null))).toBeNull();
+        });
+        it("accepts plain objects (logic_instance_v2 shape)", () => {
+            const original = { outputsToAssign: [], inputs: [{ name: "x", value: "" }] };
+            expect((0, values_1.decodeV2Values)((0, values_1.encodeV2Values)(original))).toEqual(original);
+        });
+    });
+    describe("encode — options", () => {
+        it("round-trips correctly at every supported compression level", () => {
+            const v = Array.from({ length: 200 }, (_, i) => ({ id: String(i), value: "x".repeat(50) }));
+            for (let level = 1; level <= 9; level++) {
+                const blob = (0, values_1.encodeV2Values)(v, { level });
+                expect((0, values_1.decodeV2Values)(blob)).toEqual(v);
+            }
+        });
+        it("emits different byte streams at different levels (level is honored)", () => {
+            // Highly compressible payload — level=1 vs level=9 must differ in bytes,
+            // even if which is shorter is not strictly monotonic for tiny inputs.
+            const v = Array.from({ length: 500 }, (_, i) => ({ id: String(i), value: "abcde".repeat(20) }));
+            const lvl1 = (0, values_1.encodeV2Values)(v, { level: 1 });
+            const lvl9 = (0, values_1.encodeV2Values)(v, { level: 9 });
+            expect(lvl1).not.toEqual(lvl9);
+        });
+        it("treats undefined level as default (no throw, valid output)", () => {
+            const v = [{ id: "1", value: "a" }];
+            expect((0, values_1.decodeV2Values)((0, values_1.encodeV2Values)(v, {}))).toEqual(v);
+            expect((0, values_1.decodeV2Values)((0, values_1.encodeV2Values)(v, { level: undefined }))).toEqual(v);
+        });
+    });
+    describe("round-trip — real Tenon fixtures", () => {
+        // Real V2 blobs pulled from tenonworkstudio sit under
+        // tests/fixtures/v2-blobs/<table>.<sys_id>.txt. Each one must survive a
+        // decode→encode→decode pass with structural deep-equality preserved. This
+        // is the gate that lets us trust encodeV2Values for write-path use.
+        const fixtures = fs.existsSync(FIXTURES_DIR)
+            ? fs.readdirSync(FIXTURES_DIR).filter((f) => f.endsWith(".txt"))
+            : [];
+        it("fixtures directory has at least one sample per V2 table", () => {
+            expect(fixtures.length).toBeGreaterThan(0);
+            const tables = new Set(fixtures.map((f) => f.split(".")[0]));
+            // We expect samples for at least the action_instance and flow_logic_instance
+            // shapes. trigger_instance is nice-to-have but not gating.
+            expect(tables.has("sys_hub_action_instance_v2")).toBe(true);
+            expect(tables.has("sys_hub_flow_logic_instance_v2")).toBe(true);
+        });
+        fixtures.forEach((filename) => {
+            it(`round-trips: ${filename}`, () => {
+                const blob = fs.readFileSync(path.join(FIXTURES_DIR, filename), "utf8").trim();
+                const decoded = (0, values_1.decodeV2Values)(blob);
+                const reencoded = (0, values_1.encodeV2Values)(decoded);
+                const reDecoded = (0, values_1.decodeV2Values)(reencoded);
+                expect(reDecoded).toEqual(decoded);
+            });
+        });
+        it("re-encoded blobs decompress to byte-identical JSON via Node's zlib", () => {
+            // Belt-and-suspenders: our own zlib can read what we just wrote.
+            // (RFC 1952 conformance check; SN-side acceptance is a separate integration test.)
+            if (!fixtures.length)
+                return;
+            const blob = fs.readFileSync(path.join(FIXTURES_DIR, fixtures[0]), "utf8").trim();
+            const decoded = (0, values_1.decodeV2Values)(blob);
+            const reencoded = (0, values_1.encodeV2Values)(decoded);
+            const ourJson = (0, zlib_1.gunzipSync)(Buffer.from(reencoded, "base64")).toString("utf8");
+            expect(JSON.parse(ourJson)).toEqual(decoded);
+        });
+    });
 });

package/package.json

@@ -1,14 +1,13 @@
 {
   "name": "@tenonhq/dovetail-core",
-  "version": "0.0.85",
+  "version": "0.0.87",
   "description": "Next-gen file syncer",
   "license": "GPL-3.0",
   "main": "./dist/index.js",
   "author": "Multiple",
   "private": false,
   "scripts": {
-    "copy:skills": "mkdir -p skills && cp ../../skills/*.md ./skills/",
-    "prepack": "npm run copy:skills && tsc && npm run validate-dist",
+    "prepack": "tsc && npm run validate-dist",
     "validate-dist": "node -e \"var fs=require('fs');if(!fs.existsSync('./dist/index.js')){console.error('FATAL: dist/index.js missing after build');process.exit(1)}\"",
     "test": "jest --ci --reporters=jest-junit --reporters=default --coverage --coverageReporters=cobertura --coverageReporters=html",
     "version:bump": "node ./../../Scripts/bump-version.js ./package.json",