clay-server 5.2.0-beta.1 → 5.2.0-beta.11

package/lib/dm-digest-cursor.js

@@ -0,0 +1,120 @@
+var CONVERSATION_CAP = 6000;
+
+// SDK text is persisted as delta before onTurnDone; the response preview
+// is only a notification excerpt. Historical complete text entries remain readable.
+function isDigestEntry(entry) {
+  return entry && (entry.type === "user_message" || entry.type === "assistant_message" || entry.type === "delta" || entry.type === "text");
+}
+
+function copyCursor(cursor) {
+  return {
+    historyIndex: Math.max(0, cursor.historyIndex || 0),
+    textOffset: Math.max(0, cursor.textOffset || 0),
+  };
+}
+
+function readState(history) {
+  var state = { cursor: { historyIndex: 0, textOffset: 0 } };
+  for (var i = history.length - 1; i >= 0; i--) {
+    var entry = history[i];
+    if (!entry || entry.type !== "digest_checkpoint") continue;
+    if (entry.cursor && typeof entry.cursor.historyIndex === "number") {
+      state.cursor = copyCursor(entry.cursor);
+    } else if (typeof entry.digestedIndex === "number") {
+      // Legacy checkpoints pointed at the next history entry and had no
+      // within-entry progress. Treat them as fully consumed entries.
+      state.cursor = { historyIndex: Math.max(0, entry.digestedIndex), textOffset: 0 };
+    }
+    break;
+  }
+  return state;
+}
+
+function readCursor(history) {
+  return readState(history).cursor;
+}
+
+function sameCursor(left, right) {
+  return left.historyIndex === right.historyIndex && left.textOffset === right.textOffset;
+}
+
+function safeTextEnd(text, start, maximum) {
+  var end = Math.min(text.length, start + maximum);
+  if (end < text.length && end > start) {
+    var previous = text.charCodeAt(end - 1);
+    if (previous >= 0xD800 && previous <= 0xDBFF) end--;
+  }
+  // The normal cap leaves enough room for a code point. Keep this fallback
+  // defensive so a future smaller cap cannot make the cursor stall.
+  if (end === start && end < text.length) {
+    end = start + (text.charCodeAt(start) >= 0xD800 && text.charCodeAt(start) <= 0xDBFF ? 2 : 1);
+  }
+  return Math.min(end, text.length);
+}
+
+function collectChunk(history, startCursor) {
+  var cursor = copyCursor(startCursor);
+  var parts = [];
+  var totalLength = 0;
+  var hadHistoryProgress = false;
+  var partCount = 0;
+
+  function appendPart(value) {
+    var separatorLength = partCount > 0 ? 1 : 0;
+    if (totalLength + separatorLength + value.length > CONVERSATION_CAP) return false;
+    parts.push(value);
+    totalLength += separatorLength + value.length;
+    partCount++;
+    return true;
+  }
+
+  while (cursor.historyIndex < history.length) {
+    var entry = history[cursor.historyIndex];
+    if (!isDigestEntry(entry) || typeof entry.text !== "string" || !entry.text) {
+      cursor.historyIndex++;
+      cursor.textOffset = 0;
+      hadHistoryProgress = true;
+      continue;
+    }
+
+    var text = entry.text;
+    var offset = Math.min(cursor.textOffset, text.length);
+
+    var label = entry.type === "user_message" ? "User: " : "Mate: ";
+    var available = CONVERSATION_CAP - totalLength - (partCount > 0 ? 1 : 0) - label.length;
+    if (available <= 0 && partCount > 0) break;
+    if (available <= 0) available = CONVERSATION_CAP - label.length;
+
+    if (offset < text.length) {
+      var end = safeTextEnd(text, offset, available);
+      var value = label + text.substring(offset, end);
+      if (!appendPart(value)) break;
+      cursor.textOffset = end;
+      hadHistoryProgress = true;
+      if (end < text.length) break;
+    }
+
+    cursor.historyIndex++;
+    cursor.textOffset = 0;
+  }
+
+  return {
+    conversationContent: parts.join("\n"),
+    cursor: cursor,
+    hasHistoryProgress: hadHistoryProgress,
+  };
+}
+
+function hasPending(history, cursor) {
+  var probe = collectChunk(history, cursor, "");
+  return !!probe.conversationContent;
+}
+
+module.exports = {
+  CONVERSATION_CAP: CONVERSATION_CAP,
+  collectChunk: collectChunk,
+  hasPending: hasPending,
+  readCursor: readCursor,
+  readState: readState,
+  sameCursor: sameCursor,
+};

package/lib/email-mcp-server.js

@@ -37,7 +37,7 @@ function buildShape(props, required) {
 //   appendAuditLog(entry) -> void
 function create(deps) {
   var sdk;
-  try { sdk = require("@anthropic-ai/claude-agent-sdk"); } catch (e) {
+  try { sdk = require("./yoke/sdk-server-factory").load(); } catch (e) {
     console.error("[email-mcp] Failed to load SDK:", e.message);
     return null;
   }

package/lib/github-star-count.js

@@ -0,0 +1,147 @@
+// github-star-count.js - Cached public star count for the fixed Clay repository.
+//
+// Server-wide and unauthenticated toward GitHub: one fixed public metadata
+// URL, no credentials, no redirects, no user data. A fresh value is reused for
+// an hour, concurrent callers share one request, failures back off, and the
+// last successful count survives a failed refresh. Unknown stays null; a count
+// is never invented.
+
+var https = require("https");
+
+var REPO = "chadbyte/clay";
+var HOST = "api.github.com";
+var PATH = "/repos/" + REPO;
+var TTL_MS = 60 * 60 * 1000;
+var TIMEOUT_MS = 5000;
+var BACKOFF_BASE_MS = 60 * 1000;
+var MAX_BODY_BYTES = 256 * 1024;
+
+// Resolves { status, body } or rejects. Never follows redirects. `timeoutMs`
+// is a wall-clock deadline for the whole exchange (DNS, TLS, headers, body),
+// not socket inactivity, so a slow trickle cannot pin the request. `get` is
+// injectable for tests.
+function fetchRepoMetadata(timeoutMs, get) {
+  get = get || https.get;
+  return new Promise(function (resolve, reject) {
+    var settled = false;
+    var req = null;
+    var deadline = null;
+    function finish(fn, value) {
+      if (settled) return;
+      settled = true;
+      clearTimeout(deadline);
+      fn(value);
+    }
+    function fail(err) {
+      finish(reject, err);
+      if (req) req.destroy();
+    }
+    deadline = setTimeout(function () { fail(new Error("timeout")); }, timeoutMs);
+    try {
+      req = get({
+        host: HOST,
+        path: PATH,
+        method: "GET",
+        headers: {
+          "User-Agent": "clay-server",
+          "Accept": "application/vnd.github+json",
+          "X-GitHub-Api-Version": "2022-11-28",
+        },
+      }, function (res) {
+        var chunks = [];
+        var size = 0;
+        res.on("data", function (chunk) {
+          size += chunk.length;
+          if (size > MAX_BODY_BYTES) {
+            fail(new Error("response too large"));
+            if (res.destroy) res.destroy();
+            return;
+          }
+          chunks.push(chunk);
+        });
+        res.on("end", function () {
+          finish(resolve, { status: res.statusCode, body: Buffer.concat(chunks).toString("utf8") });
+        });
+        res.on("error", fail);
+        res.on("aborted", function () { fail(new Error("aborted")); });
+      });
+    } catch (err) {
+      fail(err);
+      return;
+    }
+    req.on("error", fail);
+    if (settled) req.destroy();
+  });
+}
+
+function parseStars(response) {
+  if (!response || response.status !== 200) return null;
+  var data;
+  try { data = JSON.parse(response.body); } catch (e) { return null; }
+  var n = data && data.stargazers_count;
+  if (typeof n !== "number" || !isFinite(n) || n < 0 || Math.floor(n) !== n) return null;
+  return n;
+}
+
+// opts (all optional, for tests): fetch(timeoutMs) -> Promise<{status, body}>,
+// now() -> ms, ttlMs, timeoutMs, backoffBaseMs.
+function createGithubStarCount(opts) {
+  opts = opts || {};
+  var fetchFn = opts.fetch || fetchRepoMetadata;
+  var now = opts.now || Date.now;
+  var ttlMs = opts.ttlMs || TTL_MS;
+  var timeoutMs = opts.timeoutMs || TIMEOUT_MS;
+  var backoffBaseMs = opts.backoffBaseMs || BACKOFF_BASE_MS;
+
+  var stars = null;
+  var fetchedAt = null;
+  var failures = 0;
+  var retryAt = 0;
+  var inflight = null;
+
+  function snapshot() {
+    return { repo: REPO, stars: stars, fetchedAt: fetchedAt };
+  }
+
+  function refresh() {
+    var started;
+    try {
+      started = Promise.resolve(fetchFn(timeoutMs));
+    } catch (e) {
+      started = Promise.reject(e);
+    }
+    return started.then(function (response) {
+      var parsed = parseStars(response);
+      if (parsed === null) throw new Error("unusable response");
+      stars = parsed;
+      fetchedAt = now();
+      failures = 0;
+      retryAt = 0;
+    }).catch(function () {
+      failures++;
+      retryAt = now() + Math.min(backoffBaseMs * Math.pow(2, failures - 1), ttlMs);
+    }).then(function () {
+      inflight = null;
+      return snapshot();
+    });
+  }
+
+  function get() {
+    if (stars !== null && fetchedAt !== null && now() - fetchedAt < ttlMs) return Promise.resolve(snapshot());
+    if (inflight) return inflight;
+    if (now() < retryAt) return Promise.resolve(snapshot());
+    inflight = refresh();
+    return inflight;
+  }
+
+  return { get: get };
+}
+
+module.exports = {
+  createGithubStarCount: createGithubStarCount,
+  parseStars: parseStars,
+  fetchRepoMetadata: fetchRepoMetadata,
+  MAX_BODY_BYTES: MAX_BODY_BYTES,
+  REPO: REPO,
+  TTL_MS: TTL_MS,
+};

package/lib/issues-service.js

@@ -129,25 +129,35 @@ function attachIssuesService(ctx) {
       principal: principal, worktreeOnly: authorization.worktreeOnly };
   }
 
-  function bind(source, isSession) {
+  function bind(source, isSession, canonicalClay) {
     if (!source || typeof source.projectSlug !== "string") return null;
     var initialSlug = source.projectSlug;
     var initialProject = projectFor(initialSlug);
     var initialSession = isSession ? source.session : null;
+    var clayInitial;
+    if (canonicalClay) { try { clayInitial = require("./canonical-clay-record-authority").resolve(source.authority, initialSlug); } catch (error) { return null; } }
     if (!initialProject || (isSession && !initialSession)) return null;
     if (initialSession) provenance.ensureOrigin(initialSession);
-    var initialUserId = initialSession ? initialSession.ownerId || null : source.userId || null;
+    var initialUserId = initialSession ? initialSession.ownerId || null : canonicalClay ? clayInitial.principal.userId || clayInitial.principal.singleUserOwnerId : source.userId || null;
     var initialOrigin = initialSession ? initialSession.sessionOriginId || null : null;
     var initial = bindingState(initialProject, initialSession, initialUserId, initialOrigin);
     if (!initial) return null;
 
     function current() {
       if (projectFor(initialSlug) !== initialProject) throw new Error("This Issues binding is no longer valid.");
+      if (canonicalClay) {
+        var clayLive = require("./canonical-clay-record-authority").resolve(source.authority, initialSlug);
+        if (clayLive.project !== initialProject || clayLive.status.path !== clayInitial.status.path || clayLive.status.projectKnowledgeId !== clayInitial.status.projectKnowledgeId || clayLive.status.changeSetId !== clayInitial.status.changeSetId) throw new Error("Selected Issues context changed.");
+      }
       var live = bindingState(initialProject, initialSession, initialUserId, initialOrigin);
       if (!live) throw new Error("This Issues binding is no longer valid.");
       return live;
     }
 
+    function expected(entry, args) { if (canonicalClay && (!Number.isInteger(args.expectedRevision) || entry.revision !== args.expectedRevision)) throw new Error("Issue revision conflict."); }
+
+    function actor(state) { return canonicalClay ? require("./canonical-clay-record-authority").resolve(source.authority, initialSlug).actor : author(state.user, state.session ? "session" : "user", state.session); }
+
     function visibleIssue(store, ref, state) {
       var result = store.read(ref);
       if (state.worktreeOnly && (!result.context || result.context.changeSetId !== state.status.changeSetId)) throw new Error("Issue not found.");
@@ -174,7 +184,7 @@ function attachIssuesService(ctx) {
         path: state.status.path, knowledgeId: governing.projectKnowledgeId,
         changeSetId: state.status.changeSetId || null, worktreeOnly: state.worktreeOnly,
         recordContext: clone(issueContext(state.status)),
-        actor: clone(author(state.user, state.session ? "session" : "user", state.session)),
+        actor: clone(actor(state)),
         osIdentity: needsIdentity ? osIdentity(state) : null };
     }
 
@@ -184,7 +194,7 @@ function attachIssuesService(ctx) {
       var liveIdentity = needsIdentity ? osIdentity(fresh) : null;
       if (fresh.project !== before.project || fresh.session !== before.session || (fresh.user.id || null) !== before.userId ||
           fresh.status.path !== before.path || !governing || governing.projectKnowledgeId !== before.knowledgeId ||
-          (fresh.status.changeSetId || null) !== before.changeSetId || fresh.worktreeOnly !== before.worktreeOnly ||
+          (fresh.status.changeSetId || null) !== before.changeSetId || fresh.worktreeOnly !== before.worktreeOnly || JSON.stringify(issueContext(fresh.status)) !== JSON.stringify(before.recordContext) ||
           JSON.stringify(liveIdentity) !== JSON.stringify(before.osIdentity)) throw new Error("Issue authorization changed during the operation.");
       return fresh;
     }
@@ -200,7 +210,7 @@ function attachIssuesService(ctx) {
     var github = require('./issues-github').attachIssueGithub({
       locks: githubLocks, assessments: githubAssessments, run: context.runGithub, analyze: source.analyzeGithub || context.analyzeGithub,
       begin: function () {
-        if (isSession) throw new Error('GitHub filing requires an explicit action in the Issues panel.');
+        if (isSession || canonicalClay) throw new Error('GitHub filing requires an explicit action in the Issues panel.');
         return operation(current(), true);
       },
       read: function (before, ref) {
@@ -217,6 +227,7 @@ function attachIssuesService(ctx) {
     return {
       searchGithubIssue: github.search, linkGithubIssue: github.link, createGithubIssue: github.create,
       linkWorkSession: function (args, session) {
+        if (canonicalClay) throw new Error("Use reviewed project assignment or follow-up for Driver execution.");
         var state = current();
         var manager = state.project.getSessionManager();
         if (!args || !Number.isInteger(args.expectedRevision)) throw new Error("expectedRevision is required.");
@@ -232,9 +243,10 @@ function attachIssuesService(ctx) {
         if (links.some(function (link) { return link.sessionId === session.sessionOriginId; })) return entry;
         var link = { sessionId: session.sessionOriginId, role: "driver", issueRevision: args.expectedRevision };
         if (args.requestId !== undefined) link.requestId = schema.text(args.requestId, "work request id", 200, true);
-        return target.update(args.ref, { linkedWorkSessions: links.concat([link]), status: "in_progress" }, args.expectedRevision, author(state.user, "user", null));
+        return target.update(args.ref, { linkedWorkSessions: links.concat([link]), status: "in_progress" }, args.expectedRevision, canonicalClay ? actor(state) : author(state.user, "user", null));
       },
       resolveWorkSession: function (args) {
+        if (canonicalClay) throw new Error("Work sessions require the existing project Driver workflow.");
         var state = current();
         var entry = visibleIssue(openStore(state.status), args.ref, state);
         if (!(entry.linkedWorkSessions || []).some(function (link) { return link.sessionId === args.sessionOriginId; })) throw new Error("Work session unavailable.");
@@ -255,9 +267,9 @@ function attachIssuesService(ctx) {
       readIssueTombstone: function (args) { var state = current(); return scopedIssue(openStore(state.status), args && args.ref, state); },
       issueHistory: function (args) { var state = current(); var store = openStore(state.status); var history = store.history(args && args.ref); if (state.worktreeOnly) { var historical = scopedIssue(store, args && args.ref, state); if (!historical.context || historical.context.changeSetId !== state.status.changeSetId) throw new Error("Issue not found."); } return history; },
       readIssueRevision: function (args) { var state = current(); var store = openStore(state.status); visibleIssue(store, args && args.ref, state); return store.readRevision(args.ref, args.revision); },
-      commentIssue: function (args) { var state = current(); var target = openStore(state.status); visibleIssue(target, args && args.ref, state); return target.comment(args.ref, args.body, author(state.user, "user", null)); },
+      commentIssue: function (args) { var state = current(); var target = openStore(state.status); expected(visibleIssue(target, args && args.ref, state), args); return target.comment(args.ref, args.body, canonicalClay ? actor(state) : author(state.user, "user", null)); },
       listIssueFeedback: function (args) {
-        if (!isSession) throw new Error("Only the Project Driver can review issue comments.");
+        if (!isSession && !canonicalClay) throw new Error("Only the Project Driver can review issue comments.");
         var state = current(); var target = openStore(state.status); var out = []; var feedbackLimit = args && args.limit === undefined ? 200 : Number(args.limit);
         if (!Number.isInteger(feedbackLimit) || feedbackLimit < 1 || feedbackLimit > 200) throw new Error("Feedback limit must be between 1 and 200.");
         var position = { issueCursor: null, issueIndex: 0, commentOffset: 0 };
@@ -297,31 +309,32 @@ function attachIssuesService(ctx) {
         return { feedback: out, total: out.length, nextCursor: nextCursor, truncated: !!nextCursor };
       },
       reviewIssueComment: function (args) {
-        if (!isSession) throw new Error("Only the Project Driver can review issue comments.");
+        if (!isSession && !canonicalClay) throw new Error("Only the Project Driver can review issue comments.");
         var state = current(); var target = openStore(state.status); var entry = visibleIssue(target, args && args.ref, state);
+        expected(entry, args);
         if (args.action === "incorporate") {
           if (!Number.isInteger(args.expectedRevision)) throw new Error("expectedRevision is required to incorporate a comment.");
           var data = cleanPublic(args, { ref: true, expectedRevision: true, commentId: true, action: true, response: true, commitSha: true }, true);
           if (data.status === "resolved") throw new Error("Use update_issue with verified commit evidence before resolving this Issue.");
-          return target.incorporate(entry.ref, args.commentId, data, args.expectedRevision, args.response, author(state.user, "session", state.session));
+          return target.incorporate(entry.ref, args.commentId, data, args.expectedRevision, args.response, actor(state));
         }
-        return target.review(entry.ref, args.commentId, args.action, args.response, author(state.user, "session", state.session));
+        return target.review(entry.ref, args.commentId, args.action, args.response, actor(state));
       },
       removeIssue: function (args) {
         var state = current();
         if (!state.principal.isProjectOwner) throw new Error("Only the project owner can delete Issues.");
         var target = openStore(state.status); var entry = visibleIssue(target, args && args.ref, state);
         if (!Number.isInteger(args && args.expectedRevision) || entry.revision !== args.expectedRevision) throw new Error("Issue revision conflict.");
-        return target.remove(entry.ref, author(state.user, "user", null), args.expectedRevision);
+        return target.remove(entry.ref, canonicalClay ? actor(state) : author(state.user, "user", null), args.expectedRevision);
       },
       createIssue: function (args) {
-        if (!isSession) throw new Error("People cannot create issues directly. Ask the Project Driver to create one.");
+        if (!isSession && !canonicalClay) throw new Error("People cannot create issues directly. Ask the Project Driver to create one.");
         var data = cleanPublic(args, { commitSha: true }, true);
         var status = data.status === undefined ? "open" : data.status;
         if (schema.STATUSES.indexOf(status) === -1) throw new Error("Invalid status.");
         var state = current();
         if (status !== "resolved") return Promise.resolve(openStore(state.status).create(data,
-          author(state.user, state.session ? "session" : "user", state.session), issueContext(state.status)));
+          actor(state), issueContext(state.status)));
         if (typeof data.resolutionSummary !== "string" || !data.resolutionSummary.trim()) throw new Error("resolutionSummary is required when resolving an issue.");
         var before = operation(state, true);
         return evidence(before, args.commitSha).then(function (proof) {
@@ -340,7 +353,7 @@ function attachIssuesService(ctx) {
         var existing = visibleIssue(target, ref, state);
         if (existing.revision !== expectedRevision) throw new Error("Issue revision conflict.");
         if (data.status !== "resolved") return Promise.resolve(target.update(ref, data, expectedRevision,
-          author(state.user, state.session ? "session" : "user", state.session)));
+          actor(state)));
         if (typeof data.resolutionSummary !== "string" || !data.resolutionSummary.trim()) throw new Error("resolutionSummary is required when resolving an issue.");
         var before = operation(state, true);
         return evidence(before, args.commitSha).then(function (proof) {
@@ -354,6 +367,7 @@ function attachIssuesService(ctx) {
   }
 
   return { bindUser: function (source) { return bind(source, false); },
+    bindCanonicalClay: function (source) { return bind(source, false, true); },
     bindProjectSession: function (source) { return bind(source, true); },
     issueScopeIdForKnowledgeId: issuesStore.issueScopeIdForKnowledgeId };
 }

package/lib/kiro-defaults.js

@@ -1,22 +1 @@
-// Kiro-specific default values. Single source of truth — do not duplicate
-// elsewhere. Consumed by the server-side vendor plumbing and sent to clients
-// via the config state so the UI renders the right controls.
-
-var KIRO_DEFAULTS = {
-  // Kiro CLI 2.18.1 exposes the next-generation agent as the v3 engine. Its
-  // general-purpose mode id is "vibe" (displayed as "Default").
-  engine: "v3",
-  mode: "vibe",
-};
-
-function getKiroConfig(sm) {
-  return {
-    engine: (sm && sm.kiroEngine) || KIRO_DEFAULTS.engine,
-    mode: (sm && sm.kiroMode) || KIRO_DEFAULTS.mode,
-  };
-}
-
-module.exports = {
-  KIRO_DEFAULTS: KIRO_DEFAULTS,
-  getKiroConfig: getKiroConfig,
-};
+module.exports = require("./yoke/options/kiro-defaults");

package/lib/knowledge-db-import-validation.js

@@ -0,0 +1,46 @@
+var crypto = require("crypto");
+var validation = require("./knowledge-db-validation");
+var MAX_BATCH_ROWS = 100;
+var MAX_BATCH_BYTES = 256 * 1024;
+
+function fail(message, code) { throw Object.assign(new Error(message), { code: code || "INVALID" }); }
+function json(value, depth) {
+  if (depth > 32) fail("Import JSON exceeds 32 nesting levels.");
+  if (value === null || typeof value === "string" || typeof value === "boolean") return;
+  if (typeof value === "number" && Number.isFinite(value)) return;
+  if (typeof value !== "object" || !value) fail("Import payload must contain finite JSON values.");
+  if (!Array.isArray(value)) validation.plainObject(value, "Import payload");
+  Object.keys(value).forEach(function (key) { json(value[key], depth + 1); });
+}
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (!value || typeof value !== "object") return value;
+  var out = {}; Object.keys(value).sort().forEach(function (key) { out[key] = canonical(value[key]); }); return out;
+}
+function identity(namespace, key) { return JSON.stringify([namespace, key]); }
+function sourceText(value, label, max) {
+  var clean = validation.text(value, label, max, false);
+  if (clean !== value) fail(label + " must not have surrounding whitespace.");
+  return clean;
+}
+function batch(args) {
+  validation.plainObject(args, "Import request");
+  var namespace = sourceText(args.sourceNamespace, "Source namespace", 120);
+  if (!Array.isArray(args.rows) || !args.rows.length || args.rows.length > MAX_BATCH_ROWS) fail("Import batch must contain between 1 and " + MAX_BATCH_ROWS + " rows.", "IMPORT_BATCH_LIMIT");
+  json(args.rows, 0);
+  if (Buffer.byteLength(JSON.stringify(args.rows), "utf8") > MAX_BATCH_BYTES) fail("Import rows exceed the " + MAX_BATCH_BYTES + " byte input limit.", "IMPORT_BYTES_LIMIT");
+  var seen = new Set();
+  var rows = args.rows.map(function (row) {
+    validation.plainObject(row, "Import row");
+    var key = sourceText(row.sourceKey, "Source row key", 200);
+    if (seen.has(key)) fail("Import batch contains duplicate source row keys.");
+    seen.add(key);
+    if (!Object.prototype.hasOwnProperty.call(row, "sourcePayload")) fail("Each import row needs an immutable sourcePayload.");
+    validation.plainObject(row.values, "Import values");
+    var fingerprint = crypto.createHash("sha256").update(JSON.stringify(canonical({ sourcePayload: row.sourcePayload, values: row.values }))).digest("hex");
+    return { sourceKey: key, fingerprint: fingerprint, values: row.values };
+  });
+  return { sourceNamespace: namespace, rows: rows };
+}
+
+module.exports = { batch: batch, identity: identity, sourceText: sourceText, MAX_BATCH_ROWS: MAX_BATCH_ROWS, MAX_BATCH_BYTES: MAX_BATCH_BYTES };

package/lib/knowledge-db-import.js

@@ -0,0 +1,99 @@
+var validation = require("./knowledge-db-validation");
+var input = require("./knowledge-db-import-validation");
+
+// importRecords({ databaseId, expectedDatabaseRevision, operationId, sourceNamespace,
+// rows: [{ sourceKey, sourcePayload, values }], actor }) creates only unseen source rows.
+// Both sourcePayload and supplied values are immutable import identity; corrections use CRUD.
+// Record references can use sourceNamespace/sourceKey instead of id, including forward
+// references within this batch. Cross-database sources must already be imported locally.
+// readImportProgress({ databaseId, sourceNamespace, offset?, limit? }) returns native ids.
+// Import mappings survive receipt expiry; replay with a fresh operation id and current revision.
+// Limits: 100 rows / 256 KiB of row JSON per batch; shared 20000-record / 16 MiB
+// aggregate bounds include archives, history, mappings and receipts (500 live / 24 hours).
+// Each fresh operation retains one receipt and uses one canonical replacement; source
+// replay preserves the revision. New rows also use the existing optional summary
+// projection write. No filesystem input is accepted.
+function attachKnowledgeDbImport(ctx) {
+  function mappings(database) { return database.importMappings || []; }
+  function index(database) {
+    var out = new Map(); mappings(database).forEach(function (mapping) { out.set(input.identity(mapping.sourceNamespace, mapping.sourceKey), mapping); }); return out;
+  }
+  function convert(values, schema, databases, next, planned) {
+    var fields = {}; schema.forEach(function (field) { fields[field.id] = field; });
+    var out = Object.assign({}, values);
+    Object.keys(out).forEach(function (fieldId) {
+      var field = fields[fieldId]; var value = out[fieldId];
+      if (!field || field.type !== "reference" || !value || !Object.prototype.hasOwnProperty.call(value, "sourceKey")) return;
+      validation.plainObject(value, "Import reference");
+      if (value.type !== "record" || value.id !== undefined) throw ctx.error("Source references require type record and no native id.", "INVALID");
+      var databaseId = validation.id(value.databaseId || field.targetDatabaseId || next._id, "reference database id");
+      var namespace = input.sourceText(value.sourceNamespace, "Reference source namespace", 120);
+      var key = input.sourceText(value.sourceKey, "Reference source key", 200);
+      var target = databases.find(function (database) { return database._id === databaseId; });
+      var mapping = databaseId === next._id ? planned.get(input.identity(namespace, key)) : target && index(target).get(input.identity(namespace, key));
+      if (!mapping) throw ctx.error("Reference source row is outside this Mate or has not been imported.", "FORBIDDEN_REFERENCE");
+      out[fieldId] = { type: "record", databaseId: databaseId, id: mapping.recordId };
+    });
+    return out;
+  }
+  function importRecords(args) {
+    // Snapshot caller-owned JSON before any asynchronous operation.
+    var revalidate = args.revalidate; var batch = ctx.clone(input.batch(args)); args = ctx.clone(args); args.revalidate = revalidate;
+    var operationId = ctx.mutationId(args.operationId); var fingerprint = ctx.fingerprint("import", args);
+    return ctx.serialize(args.databaseId, function () { return ctx.load(args.databaseId).then(async function (current) {
+      ctx.guard(args, current);
+      var duplicate = ctx.receipt(current, operationId, fingerprint);
+      if (duplicate) return Object.assign({}, duplicate, { projectionPending: current.projectionPending === true, projectionError: current.projectionError || null });
+      if (current.archivedAt) throw ctx.error("Knowledge database is archived.", "ARCHIVED", current.revision);
+      ctx.ensureRevision(args.expectedDatabaseRevision, current.revision, "database");
+      var next = ctx.clone(current); var planned = index(current); var added = [];
+      var resultMappings = batch.rows.map(function (row) {
+        var identity = input.identity(batch.sourceNamespace, row.sourceKey); var existing = planned.get(identity);
+        if (existing && existing.fingerprint !== row.fingerprint) throw ctx.error("Source row was already imported with a different payload: " + row.sourceKey, "IMPORT_SOURCE_CONFLICT");
+        if (existing) return Object.assign({}, existing, { created: false });
+        var mapping = { sourceNamespace: batch.sourceNamespace, sourceKey: row.sourceKey, fingerprint: row.fingerprint, recordId: ctx.newId("rec") };
+        planned.set(identity, mapping); added.push({ row: row, mapping: mapping });
+        return Object.assign({}, mapping, { created: true });
+      });
+      if (next.records.length + added.length > validation.MAX_RECORDS) throw ctx.error("Database exceeds the " + validation.MAX_RECORDS + " record limit (including archived records).", "LIMIT");
+      var result = { databaseId: current._id, databaseRevision: current.revision, sourceNamespace: batch.sourceNamespace, createdCount: added.length, replayedCount: batch.rows.length - added.length, mappings: resultMappings };
+      // Remember fresh replay operations too, so reusing an operation id with another
+      // payload still conflicts. Preserve record values, archives and database revision.
+      if (!added.length) {
+        ctx.remember(next, operationId, fingerprint, result);
+        await ctx.replace(current, next, args);
+        return Object.assign(result, { projectionPending: current.projectionPending === true, projectionError: current.projectionError || null });
+      }
+      var databases = await ctx.all(); ctx.guard(args, current); var at = Date.now(); var who = ctx.actor(args.actor);
+      added.forEach(function (item) {
+        next.records.push({ id: item.mapping.recordId, revision: 1, values: {}, archivedAt: null, createdAt: at, updatedAt: at, createdBy: who, updatedBy: who });
+        next.recordHistory[item.mapping.recordId] = [];
+      });
+      var localDatabases = databases.filter(function (database) { return database._id !== next._id; }).concat([next]);
+      var context = ctx.referenceContext(localDatabases, next);
+      added.forEach(function (item, position) {
+        var values = convert(item.row.values, current.schema, databases, next, planned);
+        next.records[current.records.length + position].values = validation.normalizeValues(values, current.schema, context, null);
+      });
+      next.importMappings = mappings(next).concat(added.map(function (item) { return item.mapping; }));
+      next.revision++; next.updatedAt = at; next.updatedBy = who; next.projectionPending = true; next.projectionError = null;
+      result.databaseRevision = next.revision;
+      ctx.remember(next, operationId, fingerprint, result);
+      await ctx.replace(current, next, args);
+      return ctx.finish(next, result, who);
+    }); });
+  }
+  function readImportProgress(args) {
+    var namespace = input.sourceText(args.sourceNamespace, "Source namespace", 120);
+    var offset = args.offset === undefined ? 0 : args.offset; var limit = args.limit === undefined ? 100 : args.limit;
+    if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > 100) throw ctx.error("Import progress requires a nonnegative integer offset and limit between 1 and 100.", "INVALID");
+    return ctx.load(args.databaseId).then(function (database) {
+      var rows = mappings(database).filter(function (mapping) { return mapping.sourceNamespace === namespace; });
+      rows.sort(function (a, b) { return a.sourceKey === b.sourceKey ? 0 : a.sourceKey < b.sourceKey ? -1 : 1; });
+      return { databaseId: database._id, databaseRevision: database.revision, sourceNamespace: namespace, total: rows.length, mappings: ctx.clone(rows.slice(offset, offset + limit)), nextOffset: offset + limit < rows.length ? offset + limit : null };
+    });
+  }
+  return { importRecords: importRecords, readImportProgress: readImportProgress };
+}
+
+module.exports = { attachKnowledgeDbImport: attachKnowledgeDbImport, MAX_BATCH_ROWS: input.MAX_BATCH_ROWS, MAX_BATCH_BYTES: input.MAX_BATCH_BYTES };