@bragi-gmbh/codebus 1.6.5 → 1.6.7

package/dist/kit/graph-publish.mjs

@@ -79,15 +79,20 @@
  *                              against a fixture, per the build ticket).
  *   --skip-build             — same effect, reads graphify-out/graph.json
  *                              as-is without re-running graphify.
- *   --dry-run                — do everything except the final
- *                              graph_ingest_with_token call (still reports
- *                              the head and still validates).
+  *   --dry-run                — do everything except the final
+  *                              graph_ingest_with_token call (still reports
+  *                              the head and still validates). Reports the
+  *                              post-compaction size against the 25 MB
+  *                              ingest cap (PASS/FAIL) so pre-builds know.
+  *   --no-compact              — disable artifact compaction (compaction is
+  *                              ON by default).
  */
 
 import { execFileSync } from "node:child_process";
 import { existsSync, readFileSync, realpathSync } from "node:fs";
 import { join, resolve, relative, isAbsolute } from "node:path";
 import { pathToFileURL } from "node:url";
+import { createHash } from "node:crypto";
 import { createClient } from "@supabase/supabase-js";
 
 const args = process.argv.slice(2);
@@ -97,7 +102,233 @@ const opt = (name) => {
   return i >= 0 ? args[i + 1] : undefined;
 };
 
+// Artifact size cap + compaction (25 MB ingest cap, 0050
+// graph_ingest_artifact: octet_length(p_artifact::text) > 25MB rejects).
+// EXACT stored/read field map, derived from the RPC bodies — the compactor
+// below must keep every one of these byte-identical and may drop anything
+// else:
+//
+// 0050 graph_ingest_validate: nodes[] needs id/label/file_type (+optional
+//   tier); links[] (or edges[]) needs source/target/relation/confidence
+//   (+owner_source when relation=owned_by); hyperedges[] needs
+//   id/label/relation/confidence + non-empty nodes[] (+optional tier via
+//   confidence). Top level needs nodes + links/edges arrays (hyperedges
+//   optional).
+// 0063 graph_content_sync (the ONLY projection into graph_nodes/graph_edges,
+//   full-replace on each promoted ingest): nodes keep id, label, file_type,
+//   source_file, source_location, community, norm_label, _origin, tier,
+//   metadata; edges keep source, target, relation, confidence,
+//   confidence_score, weight, source_file, context, owner_source.
+//   Hyperedges are NOT projected into tables but ARE persisted verbatim in
+//   the content-addressed graph_artifacts row — kept whole.
+// 20260922230000 graph_read_contract_v1 (signature/doc derivation): reads
+//   ONLY meta->>signature/def and meta->>doc/docstring/description, i.e.
+//   keys INSIDE the node's metadata object — metadata is preserved whole,
+//   so the derivation is unaffected.
+//
+// Everything else observed in real artifacts is dropped: top-level extras
+// (directed, multigraph, graph — which DUPLICATES hyperedges —,
+// built_at_commit), stray per-node keys (e.g. source_url/captured_at/
+// author/contributor), per-edge source_location (never read or stored).
+// Minification comes free via JSON.stringify (no whitespace).
+//
+// Exported for scripts/test/graph-compact.validate.test.mjs (same
+// import.meta.url guard convention as resolveRepoRoot above).
+export const GRAPH_ARTIFACT_SIZE_CAP_BYTES = 25 * 1024 * 1024;
+export const GRAPH_COMPACT_KEEP_NODE_FIELDS = new Set([
+  "id", "label", "file_type", "source_file", "source_location",
+  "community", "norm_label", "_origin", "tier", "metadata",
+]);
+export const GRAPH_COMPACT_KEEP_EDGE_FIELDS = new Set([
+  "source", "target", "relation", "confidence", "confidence_score",
+  "weight", "source_file", "context", "owner_source",
+]);
+
+export function compactArtifact(artifact) {
+  const removedBytes = {};
+  const dropStr = (obj, keepSet, section) => {
+    const out = {};
+    for (const [k, v] of Object.entries(obj)) {
+      if (keepSet.has(k)) {
+        out[k] = v;
+      } else {
+        const b = Buffer.byteLength(JSON.stringify({ [k]: v }), "utf-8");
+        removedBytes[`${section}.${k}`] = (removedBytes[`${section}.${k}`] ?? 0) + b;
+      }
+    }
+    return out;
+  };
+  const compacted = {};
+  // Top level: keep exactly what the RPCs read. Prefer `links` (graphify's
+  // real key); keep `edges` too if present so a future rename is not
+  // penalized — same acceptance rule as graph_ingest_validate.
+  for (const [k, v] of Object.entries(artifact)) {
+    if (k === "nodes" || k === "links" || k === "edges" || k === "hyperedges") {
+      compacted[k] = v;
+    } else {
+      const b = Buffer.byteLength(JSON.stringify({ [k]: v }), "utf-8");
+      removedBytes[`top.${k}`] = (removedBytes[`top.${k}`] ?? 0) + b;
+    }
+  }
+  if (Array.isArray(compacted.nodes)) {
+    compacted.nodes = compacted.nodes.map((n) => dropStr(n, GRAPH_COMPACT_KEEP_NODE_FIELDS, "node"));
+  }
+  for (const key of ["links", "edges"]) {
+    if (Array.isArray(compacted[key])) {
+      compacted[key] = compacted[key].map((e) => dropStr(e, GRAPH_COMPACT_KEEP_EDGE_FIELDS, "edge"));
+    }
+  }
+  // Hyperedges are persisted verbatim (content-addressed artifact) — never
+  // stripped, only minified by the final stringify.
+  return { compacted, removedBytes };
+}
+
+export function artifactBytes(obj) {
+  return Buffer.byteLength(JSON.stringify(obj), "utf-8");
+}
+
+export const GRAPH_CHUNK_PART_MAX_BYTES = 8 * 1024 * 1024;
+
+// buildCanonicalText: THE canonical serialization for chunked ingest.
+// Returns { head, chunks, str, sha256, counts, meta } where
+//   str = head + chunks.join("") + '}'
+// byte-identically to what graph_ingest_commit_with_token reassembles
+// (head + verbatim-concatenated textchunk payloads + '}'):
+//   head = {"directed":<d>,"multigraph":<m>,"graph":<g>,
+// chunks are verbatim slices of the body
+// '"nodes":[...],"links":[...],"hyperedges":[...]' (WITHOUT the trailing
+// brace), split only at element boundaries (never mid-codepoint: element
+// granularity via JSON.stringify per element). Inter-element and
+// inter-section commas live INSIDE the chunk bytes — no join separator is
+// added on either side. sha256 is over str. The commit RPC verifies sha
+// BEFORE parsing, so these bytes round-trip untouched (no jsonb ::text
+// re-serialization involved).
+//
+// Number-format contract: JSON.stringify emits integral floats as "1" (not
+// "1.0"), matching Postgres jsonb numeric output; the server never
+// re-serializes before hashing, so client bytes are authoritative.
+export function buildCanonicalText(artifact, maxBytes = GRAPH_CHUNK_PART_MAX_BYTES) {
+  const links = Array.isArray(artifact.links) ? artifact.links
+    : Array.isArray(artifact.edges) ? artifact.edges : [];
+  const nodes = Array.isArray(artifact.nodes) ? artifact.nodes : [];
+  const hyperedges = Array.isArray(artifact.hyperedges) ? artifact.hyperedges : [];
+  const head =
+    '{"directed":' + JSON.stringify(artifact.directed ?? false) +
+    ',"multigraph":' + JSON.stringify(artifact.multigraph ?? false) +
+    ',"graph":' + JSON.stringify(artifact.graph ?? {}) + ',';
+  // Split one JSON array's elements into verbatim slices <= maxBytes.
+  // MIDDLE slices (neither first nor last of their array) are bare element
+  // runs with NO key prefix and NO closing bracket — they are only valid
+  // concatenated between their siblings. The FIRST slice carries the
+  // '"name":[' prefix; the LAST slice carries the ']' suffix; a single-slice
+  // array carries both. Slices therefore must NEVER merge across arrays:
+  // each array's slices form one ordered group, groups join with ','.
+  const splitArray = (name, elements) => {
+    const prefix = '"' + name + '":[';
+    if (elements.length === 0) return [prefix + "]"];
+    const slices = [];
+    let cur = prefix, first = true;
+    for (const el of elements) {
+      const piece = (first ? "" : ",") + JSON.stringify(el);
+      if (!first && Buffer.byteLength(cur + piece + "]", "utf-8") > maxBytes) {
+        slices.push(cur);
+        cur = JSON.stringify(el);
+        first = false;
+        continue;
+      }
+      cur += piece;
+      first = false;
+    }
+    slices.push(cur + "]");
+    return slices;
+  };
+  // Groups join with ',' — the comma is OUTSIDE the chunk bytes (it is the
+  // join separator on both sides: server comma-joins too)... NO — verbatim
+  // protocol says no separators are added. Resolve: keep commas inside by
+  // attaching the trailing comma to each group except the last: group text
+  // is slices.join(",") and groups are concatenated directly.
+  const groups = [splitArray("nodes", nodes), splitArray("links", links), splitArray("hyperedges", hyperedges)];
+  const parts = [];
+  groups.forEach((slices, gi) => {
+    const lastGroup = gi === groups.length - 1;
+    slices.forEach((sl, i) => {
+      const lastSlice = i === slices.length - 1;
+      // Attach inter-group comma to the group's final slice (except the
+      // very last slice overall); inter-slice commas within a group attach
+      // to each slice except the group's last.
+      let chunk = sl;
+      if (!lastSlice) chunk = chunk + ",";
+      else if (!lastGroup) chunk = chunk + ",";
+      parts.push(chunk);
+    });
+  });
+  // Merge SMALL adjacent chunks (verbatim concat stays valid: commas are
+  // inside the bytes, so plain concatenation preserves the text).
+  const merged = [];
+  for (const c of parts) {
+    if (merged.length > 0 && Buffer.byteLength(merged[merged.length - 1] + c, "utf-8") <= maxBytes) {
+      merged[merged.length - 1] += c;
+    } else merged.push(c);
+  }
+  const body = merged.join("");
+  const str = head + body + "}";
+  return {
+    head,
+    meta: { directed: artifact.directed ?? false, multigraph: artifact.multigraph ?? false, graph: artifact.graph ?? {} },
+    chunks: merged,
+    str,
+    sha256: createHash("sha256").update(str, "utf-8").digest("hex"),
+    counts: { nodes: nodes.length, links: links.length, hyperedges: hyperedges.length },
+  };
+  const sha256 = createHash("sha256").update(str, "utf-8").digest("hex");
+  return {
+    head,
+    meta: { directed: artifact.directed ?? false, multigraph: artifact.multigraph ?? false, graph: artifact.graph ?? {} },
+    chunks: parts,
+    str,
+    sha256,
+    counts: { nodes: nodes.length, links: links.length, hyperedges: hyperedges.length },
+  };
+}
+
+export function chunkArray(elements, maxBytes = GRAPH_CHUNK_PART_MAX_BYTES) {
+  const parts = [];
+  let cur = [];
+  let curBytes = 2; // "[]"
+  for (const el of elements) {
+    const elBytes = Buffer.byteLength(JSON.stringify(el), "utf-8");
+    const add = (cur.length === 0 ? elBytes : elBytes + 1);
+    if (cur.length > 0 && curBytes + add > maxBytes) {
+      parts.push(cur);
+      cur = [];
+      curBytes = 2;
+    }
+    cur.push(el);
+    curBytes += cur.length === 1 ? elBytes : elBytes + 1;
+  }
+  if (cur.length > 0) parts.push(cur);
+  return parts;
+}
+
+export function buildCanonical(artifact) {
+  const links = Array.isArray(artifact.links) ? artifact.links
+    : Array.isArray(artifact.edges) ? artifact.edges : [];
+  const canonical = {
+    directed: artifact.directed ?? false,
+    multigraph: artifact.multigraph ?? false,
+    graph: artifact.graph ?? {},
+    nodes: Array.isArray(artifact.nodes) ? artifact.nodes : [],
+    links,
+    hyperedges: Array.isArray(artifact.hyperedges) ? artifact.hyperedges : [],
+  };
+  const str = JSON.stringify(canonical);
+  const sha256 = createHash("sha256").update(str, "utf-8").digest("hex");
+  return { canonical, str, sha256 };
+}
+
 const DRY_RUN = flag("--dry-run");
+const FORCE_CHUNKED = flag("--force-chunked");
+const COMPACT = !flag("--no-compact");
 const SKIP_BUILD = flag("--skip-build") || !!opt("--artifact-path");
 const ARTIFACT_PATH_OVERRIDE = opt("--artifact-path");
 
@@ -194,7 +425,12 @@ async function main() {
   // §B.2 preamble).
   if (!supabaseUrl) die("SUPABASE_URL is required");
   if (!anonKey) die("SUPABASE_ANON_KEY is required");
-  if (!ingestToken) die("GRAPH_INGEST_TOKEN is required (this repo's ingest-scoped token — mint via graph_ingest_token_mint)");
+  // --dry-run for a repo that is not chartered yet (pre-build extraction before
+  // the owner has minted a token) is legitimate: build + validate need no
+  // token. Everything that DOES need one (head report, ingest) is skipped
+  // below with an explicit line, never silently.
+  if (!ingestToken && !DRY_RUN) die("GRAPH_INGEST_TOKEN is required (this repo's ingest-scoped token — mint via graph_ingest_token_mint)");
+  if (!ingestToken && DRY_RUN) console.log("dry-run without GRAPH_INGEST_TOKEN: head report will be SKIPPED (token-less pre-build)");
 
   let commitSha = process.env.GITHUB_SHA;
   if (!commitSha) {
@@ -220,17 +456,22 @@ async function main() {
   // work happens. This is what makes freshness honest even when step 2
   // fails below — the hub already knows main advanced. Token-scoped: no
   // repo_id is sent, it's resolved server-side from GRAPH_INGEST_TOKEN.
-  section("Step 1: graph_head_report_with_token (authoritative tip, before build)");
-  {
-    const { error } = await supabase.rpc("graph_head_report_with_token", {
-      p_token: ingestToken,
-      p_sha: commitSha,
-      p_run_id: String(runId),
-    });
-    if (error) die(`graph_head_report_with_token failed: ${error.message} (nothing downstream can be trusted if the tip itself was never recorded)`);
-    console.log(`✓ head reported: sha=${commitSha}`);
+  if (!ingestToken && DRY_RUN) {
+    section("Step 1: graph_head_report_with_token — SKIPPED (dry-run, no token)");
+  } else {
+    section("Step 1: graph_head_report_with_token (authoritative tip, before build)");
+    {
+      const { error } = await supabase.rpc("graph_head_report_with_token", {
+        p_token: ingestToken,
+        p_sha: commitSha,
+        p_run_id: String(runId),
+      });
+      if (error) die(`graph_head_report_with_token failed: ${error.message} (nothing downstream can be trusted if the tip itself was never recorded)`);
+      console.log(`✓ head reported: sha=${commitSha}`);
+    }
   }
 
+
   // ── Step 2: build (unless testing against a fixture/existing artifact).
   section("Step 2: graphify build");
   const graphJsonPath = join(repoRoot, "graphify-out", "graph.json");
@@ -263,11 +504,69 @@ async function main() {
   } catch (err) {
     die(`failed to parse artifact JSON at ${artifactPath}: ${err.message}`);
   }
+  // ── Step 2b: deterministic augmentation (SQL functions, gateway
+  // routes, contract schemas as EXTRACTED nodes). Runs after the graphify
+  // build, before validation, so the SAME augmented artifact is validated
+  // (Step 4) and ingested (Step 5). Uses only vocab relations/file_types —
+  // no vocabulary change required. Persisted back to the artifact file.
+  section("Step 2b: deterministic augmentation (graph-augment.mjs)");
+  {
+    try {
+      const { augmentArtifact } =
+        await import(join(repoRoot, "scripts", "graph-augment.mjs"));
+      const res = augmentArtifact(artifact, repoRoot);
+      const { writeFileSync } = await import("node:fs");
+      writeFileSync(artifactPath, JSON.stringify(res.artifact, null, 2));
+      artifact = res.artifact;
+      console.log(`✓ augment: +${res.addedNodes} nodes +${res.addedEdges} edges ` +
+        `(sql ${res.per.sql.nodes}n/${res.per.sql.edges}e, ` +
+        `routes ${res.per.routes.nodes}n/${res.per.routes.edges}e, ` +
+        `schemas ${res.per.schemas.nodes}n/${res.per.schemas.edges}e)`);
+    } catch (err) {
+      die(`augmentation failed: ${err.message}`);
+    }
+  }
+
   const nodeCount = artifact.nodes?.length ?? 0;
   const edgeCount = (artifact.links ?? artifact.edges ?? []).length;
   const hyperedgeCount = (artifact.hyperedges ?? artifact.graph?.hyperedges ?? []).length;
+  const rawBytes = artifactBytes(artifact);
   console.log(`artifact: ${artifactPath}`);
   console.log(`  nodes=${nodeCount} edges=${edgeCount} hyperedges=${hyperedgeCount}`);
+  console.log(`  raw size: ${rawBytes} bytes (${(rawBytes / 1048576).toFixed(2)} MB)`);
+
+  // ── Step 2b: compaction (default ON, --no-compact disables). Removes
+  // every field no RPC reads or stores (see the map above) + minifies.
+  // Counts and all stored fields are preserved byte-identical.
+  if (COMPACT) {
+    const { compacted, removedBytes } = compactArtifact(artifact);
+    const compactBytes = artifactBytes(compacted);
+    const cn = compacted.nodes?.length ?? 0;
+    const ce = (compacted.links ?? compacted.edges ?? []).length;
+    const ch = (compacted.hyperedges ?? []).length;
+    if (cn !== nodeCount || ce !== edgeCount) {
+      die(`compaction changed counts (nodes ${nodeCount}->${cn}, edges ${edgeCount}->${ce}) — refusing to publish a compacted artifact that disagrees with the build`);
+    }
+    console.log(`  compacted size: ${compactBytes} bytes (${(compactBytes / 1048576).toFixed(2)} MB)`);
+    const entries = Object.entries(removedBytes).sort((a, b) => b[1] - a[1]);
+    if (entries.length > 0) {
+      console.log(`  removed fields:`);
+      for (const [k, b] of entries) {
+        console.log(`    ${k}: ${b} bytes`);
+      }
+    } else {
+      console.log(`  (compaction removed nothing — artifact already minimal)`);
+    }
+    artifact = compacted;
+  } else {
+    console.log(`  (--no-compact given: skipping compaction)`);
+  }
+  const finalBytes = artifactBytes(artifact);
+  const capState = finalBytes > GRAPH_ARTIFACT_SIZE_CAP_BYTES ? "FAIL" : "PASS";
+  console.log(`  ingest size check (${(GRAPH_ARTIFACT_SIZE_CAP_BYTES / 1048576).toFixed(0)} MB cap): ${finalBytes} bytes — ${capState}`);
+  if (capState === "FAIL") {
+    console.log(`  (artifact exceeds the 25 MB ingest cap even after compaction — ingest WILL be rejected; options: raise the cap via migration, or shard by top-level directory)`);
+  }
 
   // ── Step 3: resolve the current vocab version (never hardcode — the
   // vocabulary is the versioned artifact, per B.2.3). graph_vocab_current()
@@ -314,7 +613,54 @@ async function main() {
   // last-good serving completely untouched (0050's own guarantee), and
   // (finding 2) the exact validated artifact is persisted content-addressed
   // in the SAME transaction as promotion.
+  //
+  // Chunked path: when the minified artifact exceeds the 25 MB cap (or
+  // --force-chunked), canonicalize + split into <=8MB parts and ingest via
+  // graph_ingest_begin_with_token / graph_ingest_append_with_token /
+  // graph_ingest_commit_with_token. Single-call path unchanged otherwise.
   section("Step 5: graph_ingest_with_token (publish)");
+  const useChunked = finalBytes > GRAPH_ARTIFACT_SIZE_CAP_BYTES || FORCE_CHUNKED;
+  if (useChunked) {
+    const minified = JSON.parse(JSON.stringify(artifact));
+    const { meta, chunks, str, sha256, counts } = buildCanonicalText(minified);
+    const totalBytes = Buffer.byteLength(str, "utf-8");
+    console.log(`chunked ingest: sha=${sha256} textchunks=${chunks.length} nodes=${counts.nodes} links=${counts.links} hyperedges=${counts.hyperedges} total=${totalBytes} bytes`);
+    if (DRY_RUN) { console.log("=== DRY-RUN: skipping chunked ingest RPCs ==="); process.exit(0); }
+    const { data: beginData, error: beginErr } = await supabase.rpc("graph_ingest_begin_with_token", {
+      p_token: ingestToken, p_source_sha: commitSha,
+      p_run_id: String(runId), p_workflow_ref: workflowRef,
+      p_meta: { vocab_version: vocabVersion, compacted: COMPACT },
+      p_declared_node_count: counts.nodes,
+      p_declared_link_count: counts.links,
+      p_declared_hyperedge_count: counts.hyperedges,
+      p_sha256: sha256,
+    });
+    if (beginErr) die(`CHUNKED BEGIN FAILED: ${beginErr.message}`);
+    const uploadId = beginData.upload_id ?? beginData;
+    let partNo = 0;
+    const append = async (kind, part) => {
+      partNo++;
+      const bytes = Buffer.byteLength(JSON.stringify(part), "utf-8");
+      const { error } = await supabase.rpc("graph_ingest_append_with_token", {
+        p_token: ingestToken, p_upload_id: uploadId, p_part_no: partNo, p_kind: kind, p_payload: part,
+      });
+      if (error) die(`CHUNKED APPEND FAILED part ${partNo} (${kind}): ${error.message}`);
+      console.log(`  part ${partNo}: kind=${kind} bytes=${bytes}`);
+    };
+    await append("meta", meta);
+    for (const c of chunks) await append("textchunk", c);
+    // NOTE: append() JSON-encodes the chunk string as a jsonb string payload;
+    // the server concatenates chunk text verbatim (payload #>> '{}').
+    const { data: commitData, error: commitErr } = await supabase.rpc("graph_ingest_commit_with_token", {
+      p_token: ingestToken, p_upload_id: uploadId,
+    });
+    if (commitErr) die(`CHUNKED COMMIT FAILED: ${commitErr.message}`);
+    console.log(`chunked ingest committed: parts=${partNo} total=${totalBytes} bytes`);
+    console.log(JSON.stringify(commitData));
+    console.log("");
+    console.log("=== SUCCESS ===");
+    return;
+  }
   {
     const { data, error } = await supabase.rpc("graph_ingest_with_token", {
       p_token: ingestToken,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bragi-gmbh/codebus",
-  "version": "1.6.5",
+  "version": "1.6.7",
   "type": "module",
   "bin": {
     "codebus": "bin/codebus.js",