@omgbase/core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/LICENSE +21 -0
  2. package/dist/src/cli/eval-matcher.d.ts +13 -0
  3. package/dist/src/cli/eval-matcher.js +36 -0
  4. package/dist/src/core/attach.d.ts +5 -0
  5. package/dist/src/core/attach.js +38 -0
  6. package/dist/src/core/cursor.d.ts +11 -0
  7. package/dist/src/core/cursor.js +35 -0
  8. package/dist/src/core/hash.d.ts +23 -0
  9. package/dist/src/core/hash.js +82 -0
  10. package/dist/src/core/ids.d.ts +6 -0
  11. package/dist/src/core/ids.js +26 -0
  12. package/dist/src/core/index.d.ts +1 -0
  13. package/dist/src/core/index.js +2 -0
  14. package/dist/src/core/ingest.d.ts +43 -0
  15. package/dist/src/core/ingest.js +287 -0
  16. package/dist/src/core/order-key.d.ts +8 -0
  17. package/dist/src/core/order-key.js +72 -0
  18. package/dist/src/core/parse/parse.d.ts +3 -0
  19. package/dist/src/core/parse/parse.js +104 -0
  20. package/dist/src/core/parse/render.d.ts +3 -0
  21. package/dist/src/core/parse/render.js +20 -0
  22. package/dist/src/core/parse/tree.d.ts +8 -0
  23. package/dist/src/core/parse/tree.js +43 -0
  24. package/dist/src/core/parse/types.d.ts +52 -0
  25. package/dist/src/core/parse/types.js +9 -0
  26. package/dist/src/core/read/document.d.ts +107 -0
  27. package/dist/src/core/read/document.js +191 -0
  28. package/dist/src/core/read/nodes.d.ts +47 -0
  29. package/dist/src/core/read/nodes.js +102 -0
  30. package/dist/src/core/read/outline.d.ts +12 -0
  31. package/dist/src/core/read/outline.js +76 -0
  32. package/dist/src/core/read/reader.d.ts +125 -0
  33. package/dist/src/core/read/reader.js +206 -0
  34. package/dist/src/core/read/refs.d.ts +15 -0
  35. package/dist/src/core/read/refs.js +51 -0
  36. package/dist/src/core/store/edges.d.ts +26 -0
  37. package/dist/src/core/store/edges.js +101 -0
  38. package/dist/src/core/store/fts.d.ts +10 -0
  39. package/dist/src/core/store/fts.js +22 -0
  40. package/dist/src/core/store/gc.d.ts +18 -0
  41. package/dist/src/core/store/gc.js +69 -0
  42. package/dist/src/core/store/nodes.d.ts +5 -0
  43. package/dist/src/core/store/nodes.js +57 -0
  44. package/dist/src/core/store/properties.d.ts +38 -0
  45. package/dist/src/core/store/properties.js +177 -0
  46. package/dist/src/core/store/rebuild.d.ts +3 -0
  47. package/dist/src/core/store/rebuild.js +29 -0
  48. package/dist/src/core/store/schema.d.ts +9 -0
  49. package/dist/src/core/store/schema.js +386 -0
  50. package/dist/src/core/store/sections.d.ts +2 -0
  51. package/dist/src/core/store/sections.js +29 -0
  52. package/dist/src/core/store/store.d.ts +24 -0
  53. package/dist/src/core/store/store.js +164 -0
  54. package/dist/src/core/store/writers.d.ts +52 -0
  55. package/dist/src/core/store/writers.js +89 -0
  56. package/dist/src/core/vec.d.ts +10 -0
  57. package/dist/src/core/vec.js +33 -0
  58. package/dist/src/format/adapter.d.ts +97 -0
  59. package/dist/src/format/adapter.js +14 -0
  60. package/dist/src/format/index.d.ts +6 -0
  61. package/dist/src/format/index.js +15 -0
  62. package/dist/src/format/json.d.ts +3 -0
  63. package/dist/src/format/json.js +324 -0
  64. package/dist/src/format/markdown.d.ts +5 -0
  65. package/dist/src/format/markdown.js +217 -0
  66. package/dist/src/format/registry.d.ts +6 -0
  67. package/dist/src/format/registry.js +24 -0
  68. package/dist/src/format/yaml.d.ts +5 -0
  69. package/dist/src/format/yaml.js +413 -0
  70. package/dist/src/graph/extract.d.ts +38 -0
  71. package/dist/src/graph/extract.js +242 -0
  72. package/dist/src/graph/history.d.ts +103 -0
  73. package/dist/src/graph/history.js +210 -0
  74. package/dist/src/graph/inbound-links.d.ts +33 -0
  75. package/dist/src/graph/inbound-links.js +130 -0
  76. package/dist/src/graph/link-destinations.d.ts +22 -0
  77. package/dist/src/graph/link-destinations.js +84 -0
  78. package/dist/src/graph/link-health.d.ts +79 -0
  79. package/dist/src/graph/link-health.js +146 -0
  80. package/dist/src/graph/links.d.ts +23 -0
  81. package/dist/src/graph/links.js +42 -0
  82. package/dist/src/index.d.ts +60 -0
  83. package/dist/src/index.js +70 -0
  84. package/dist/src/mcp/errors.d.ts +17 -0
  85. package/dist/src/mcp/errors.js +18 -0
  86. package/dist/src/mcp/graph.d.ts +63 -0
  87. package/dist/src/mcp/graph.js +212 -0
  88. package/dist/src/mcp/reference.d.ts +1 -0
  89. package/dist/src/mcp/reference.js +227 -0
  90. package/dist/src/mcp/server.d.ts +30 -0
  91. package/dist/src/mcp/server.js +1076 -0
  92. package/dist/src/mcp/stdio.d.ts +13 -0
  93. package/dist/src/mcp/stdio.js +24 -0
  94. package/dist/src/migrate/mrplex.d.ts +27 -0
  95. package/dist/src/migrate/mrplex.js +29 -0
  96. package/dist/src/mutate/apply.d.ts +83 -0
  97. package/dist/src/mutate/apply.js +302 -0
  98. package/dist/src/mutate/doc-store.d.ts +49 -0
  99. package/dist/src/mutate/doc-store.js +71 -0
  100. package/dist/src/mutate/docs.d.ts +60 -0
  101. package/dist/src/mutate/docs.js +232 -0
  102. package/dist/src/mutate/known-ids.d.ts +12 -0
  103. package/dist/src/mutate/known-ids.js +154 -0
  104. package/dist/src/mutate/load.d.ts +3 -0
  105. package/dist/src/mutate/load.js +66 -0
  106. package/dist/src/mutate/lower.d.ts +32 -0
  107. package/dist/src/mutate/lower.js +232 -0
  108. package/dist/src/mutate/macros.d.ts +91 -0
  109. package/dist/src/mutate/macros.js +230 -0
  110. package/dist/src/mutate/ops.d.ts +40 -0
  111. package/dist/src/mutate/ops.js +513 -0
  112. package/dist/src/mutate/opset.d.ts +92 -0
  113. package/dist/src/mutate/opset.js +79 -0
  114. package/dist/src/mutate/plan-update.d.ts +51 -0
  115. package/dist/src/mutate/plan-update.js +187 -0
  116. package/dist/src/mutate/tree.d.ts +46 -0
  117. package/dist/src/mutate/tree.js +167 -0
  118. package/dist/src/oqx/run.d.ts +1 -0
  119. package/dist/src/oqx/run.js +6 -0
  120. package/dist/src/oqx-js/context.d.ts +23 -0
  121. package/dist/src/oqx-js/context.js +564 -0
  122. package/dist/src/oqx-js/planner.d.ts +10 -0
  123. package/dist/src/oqx-js/planner.js +91 -0
  124. package/dist/src/oqx-js/run.d.ts +41 -0
  125. package/dist/src/oqx-js/run.js +332 -0
  126. package/dist/src/oqx-js/sql/translate.d.ts +27 -0
  127. package/dist/src/oqx-js/sql/translate.js +290 -0
  128. package/dist/src/reconcile/crossdoc.d.ts +34 -0
  129. package/dist/src/reconcile/crossdoc.js +76 -0
  130. package/dist/src/reconcile/eval/generator.d.ts +21 -0
  131. package/dist/src/reconcile/eval/generator.js +129 -0
  132. package/dist/src/reconcile/eval/metrics.d.ts +32 -0
  133. package/dist/src/reconcile/eval/metrics.js +74 -0
  134. package/dist/src/reconcile/flatten.d.ts +12 -0
  135. package/dist/src/reconcile/flatten.js +37 -0
  136. package/dist/src/reconcile/phase5.d.ts +8 -0
  137. package/dist/src/reconcile/phase5.js +167 -0
  138. package/dist/src/reconcile/phase6a.d.ts +2 -0
  139. package/dist/src/reconcile/phase6a.js +159 -0
  140. package/dist/src/reconcile/phases.d.ts +28 -0
  141. package/dist/src/reconcile/phases.js +0 -0
  142. package/dist/src/reconcile/reconcile.d.ts +21 -0
  143. package/dist/src/reconcile/reconcile.js +115 -0
  144. package/dist/src/reconcile/similarity.d.ts +7 -0
  145. package/dist/src/reconcile/similarity.js +41 -0
  146. package/dist/src/reconcile/types.d.ts +46 -0
  147. package/dist/src/reconcile/types.js +19 -0
  148. package/dist/src/search/cel/ast.d.ts +74 -0
  149. package/dist/src/search/cel/ast.js +3 -0
  150. package/dist/src/search/cel/compile.d.ts +60 -0
  151. package/dist/src/search/cel/compile.js +836 -0
  152. package/dist/src/search/cel/lexer.d.ts +9 -0
  153. package/dist/src/search/cel/lexer.js +139 -0
  154. package/dist/src/search/cel/parser.d.ts +7 -0
  155. package/dist/src/search/cel/parser.js +219 -0
  156. package/dist/src/search/drain.d.ts +39 -0
  157. package/dist/src/search/drain.js +97 -0
  158. package/dist/src/search/embeddings.d.ts +131 -0
  159. package/dist/src/search/embeddings.js +263 -0
  160. package/dist/src/search/external.d.ts +13 -0
  161. package/dist/src/search/external.js +177 -0
  162. package/dist/src/search/fts-query.d.ts +1 -0
  163. package/dist/src/search/fts-query.js +64 -0
  164. package/dist/src/search/provider.d.ts +17 -0
  165. package/dist/src/search/provider.js +13 -0
  166. package/dist/src/search/resolve.d.ts +18 -0
  167. package/dist/src/search/resolve.js +23 -0
  168. package/dist/src/search/rrf.d.ts +36 -0
  169. package/dist/src/search/rrf.js +84 -0
  170. package/dist/src/search/tasks.d.ts +12 -0
  171. package/dist/src/search/tasks.js +131 -0
  172. package/dist/src/search/text.d.ts +16 -0
  173. package/dist/src/search/text.js +21 -0
  174. package/dist/src/search/vector.d.ts +18 -0
  175. package/dist/src/search/vector.js +49 -0
  176. package/dist/src/sync/admin.d.ts +59 -0
  177. package/dist/src/sync/admin.js +53 -0
  178. package/dist/src/sync/attach.d.ts +13 -0
  179. package/dist/src/sync/attach.js +29 -0
  180. package/dist/src/sync/checkpoint.d.ts +21 -0
  181. package/dist/src/sync/checkpoint.js +52 -0
  182. package/dist/src/sync/driver.d.ts +25 -0
  183. package/dist/src/sync/driver.js +80 -0
  184. package/dist/src/sync/external-source.d.ts +14 -0
  185. package/dist/src/sync/external-source.js +137 -0
  186. package/dist/src/sync/freshness.d.ts +42 -0
  187. package/dist/src/sync/freshness.js +152 -0
  188. package/dist/src/sync/fs-util.d.ts +12 -0
  189. package/dist/src/sync/fs-util.js +55 -0
  190. package/dist/src/sync/git-heuristics.d.ts +6 -0
  191. package/dist/src/sync/git-heuristics.js +16 -0
  192. package/dist/src/sync/observe.d.ts +63 -0
  193. package/dist/src/sync/observe.js +84 -0
  194. package/dist/src/sync/plugin.d.ts +54 -0
  195. package/dist/src/sync/plugin.js +12 -0
  196. package/dist/src/sync/reconciling-ingest.d.ts +12 -0
  197. package/dist/src/sync/reconciling-ingest.js +209 -0
  198. package/dist/src/sync/recovery.d.ts +6 -0
  199. package/dist/src/sync/recovery.js +34 -0
  200. package/dist/src/sync/settings.d.ts +18 -0
  201. package/dist/src/sync/settings.js +58 -0
  202. package/dist/src/sync/sources.d.ts +37 -0
  203. package/dist/src/sync/sources.js +75 -0
  204. package/dist/src/sync/tombstone.d.ts +9 -0
  205. package/dist/src/sync/tombstone.js +55 -0
  206. package/dist/src/sync/watch-lease.d.ts +14 -0
  207. package/dist/src/sync/watch-lease.js +91 -0
  208. package/dist/src/sync/watcher.d.ts +19 -0
  209. package/dist/src/sync/watcher.js +34 -0
  210. package/dist/src/sync/workspace.d.ts +34 -0
  211. package/dist/src/sync/workspace.js +120 -0
  212. package/dist/src/sync/writer-lock.d.ts +16 -0
  213. package/dist/src/sync/writer-lock.js +121 -0
  214. package/package.json +80 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Brendan Baldwin
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,13 @@
1
+ import { runEval } from "../reconcile/eval/metrics.js";
2
+ export interface EvalCliOptions {
3
+ n?: number;
4
+ size?: number;
5
+ intensity?: number;
6
+ }
7
+ export declare function formatReport(report: ReturnType<typeof runEval>): string;
8
+ /** Run the eval and return the report + a pass/fail against release gates. */
9
+ export declare function evalMatcher(opts?: EvalCliOptions): {
10
+ report: ReturnType<typeof runEval>;
11
+ text: string;
12
+ gatesPass: boolean;
13
+ };
@@ -0,0 +1,36 @@
1
+ import { runEval } from "../reconcile/eval/metrics.js";
2
+ import { DEFAULT_CONFIG } from "../reconcile/types.js";
3
+ export function formatReport(report) {
4
+ const lines = [];
5
+ lines.push(`matcher_v=${DEFAULT_CONFIG.matcherV} cases=${report.cases}`);
6
+ lines.push("");
7
+ const row = (name, m) => `${name.padEnd(10)} precision=${m.precision.toFixed(4)} recall=${m.recall.toFixed(4)} true=${m.truePairs} carried=${m.carriedPairs}`;
8
+ lines.push(row("OVERALL", report.overall));
9
+ for (const [cls, m] of Object.entries(report.perClass))
10
+ lines.push(row(cls, m));
11
+ return lines.join("\n");
12
+ }
13
+ /** Run the eval and return the report + a pass/fail against release gates. */
14
+ export function evalMatcher(opts = {}) {
15
+ const runOpts = { config: DEFAULT_CONFIG };
16
+ if (opts.n !== undefined)
17
+ runOpts.n = opts.n;
18
+ if (opts.size !== undefined)
19
+ runOpts.size = opts.size;
20
+ if (opts.intensity !== undefined)
21
+ runOpts.intensity = opts.intensity;
22
+ const report = runEval(runOpts);
23
+ const gatesPass = checkGates(report);
24
+ return { report, text: formatReport(report), gatesPass };
25
+ }
26
+ function checkGates(report) {
27
+ if (report.overall.precision < 0.995)
28
+ return false;
29
+ for (const cls of ["edit", "move", "reorder"]) {
30
+ const m = report.perClass[cls];
31
+ if (m && m.recall < 0.95)
32
+ return false;
33
+ }
34
+ return true;
35
+ }
36
+ //# sourceMappingURL=eval-matcher.js.map
@@ -0,0 +1,5 @@
1
+ import { Store } from "./store/store.js";
2
+ /** Create (or reuse) a repo row for `slug`, returning its id. When `rootPath` is
3
+ * given it is registered as an `fs` source + attachment, not stored on the repo.
4
+ * Pass null for a sourceless/headless repo. */
5
+ export declare function ensureRepo(store: Store, slug: string, rootPath?: string | null): string;
@@ -0,0 +1,38 @@
1
+ import { Store } from "./store/store.js";
2
+ import { mintId } from "./ids.js";
3
+ // Repo identity + fs-source registration (ADR-014). A repo owns identity +
4
+ // history, NOT a filesystem: `ensureRepo` mints the repo row, and when given a
5
+ // filesystem root it registers that as an `fs` source + attachment (the durable
6
+ // "where its bytes come from"). Ingesting a directory's files is a separate
7
+ // concern — `ingestDirectory` (sync/attach.ts) for the reconciling walk, or the
8
+ // freshness sweep / `observeOne` on the live path. The CLI entry point is
9
+ // `omg source add <dir>`; there is no `omg attach` verb.
10
+ /** Create (or reuse) a repo row for `slug`, returning its id. When `rootPath` is
11
+ * given it is registered as an `fs` source + attachment, not stored on the repo.
12
+ * Pass null for a sourceless/headless repo. */
13
+ export function ensureRepo(store, slug, rootPath = null) {
14
+ const existing = store.db.prepare("SELECT repo_id FROM repos WHERE slug = ?").get(slug);
15
+ if (existing)
16
+ return existing.repo_id;
17
+ const repoId = mintId("rp");
18
+ store.db.prepare("INSERT INTO repos (repo_id, slug) VALUES (?, ?)").run(repoId, slug);
19
+ if (rootPath)
20
+ registerFsSource(store, repoId, slug, rootPath);
21
+ return repoId;
22
+ }
23
+ /** Register (idempotently) an `fs` source at `root` and attach it to a repo.
24
+ * Inline raw SQL — kept here rather than calling sync/sources to avoid a
25
+ * core→sync import cycle; the source shape matches sync/sources exactly. */
26
+ function registerFsSource(store, repoId, slug, root) {
27
+ const db = store.db;
28
+ db.prepare("INSERT OR IGNORE INTO adapters (name, command, args) VALUES ('fs', 'omgbase-fs-adapter', '[]')").run();
29
+ const name = `${slug}-fs`;
30
+ let source = db.prepare("SELECT source_id FROM sources WHERE name = ?").get(name);
31
+ if (!source) {
32
+ const sourceId = mintId("src");
33
+ db.prepare("INSERT INTO sources (source_id, name, adapter, config, env) VALUES (?, ?, 'fs', ?, '{}')").run(sourceId, name, JSON.stringify({ root }));
34
+ source = { source_id: sourceId };
35
+ }
36
+ db.prepare("INSERT OR IGNORE INTO attachments (repo_id, source_id) VALUES (?, ?)").run(repoId, source.source_id);
37
+ }
38
+ //# sourceMappingURL=attach.js.map
@@ -0,0 +1,11 @@
1
+ /** A cursor this surface did not issue (malformed / tampered / from another
2
+ * tool). The MCP layer maps it to `filter_invalid`. */
3
+ export declare class CursorInvalid extends Error {
4
+ readonly surface: string;
5
+ constructor(surface: string);
6
+ }
7
+ /** Encode a keyset position as an opaque cursor. */
8
+ export declare function encodeCursor(parts: readonly string[]): string;
9
+ /** Decode a cursor issued by `encodeCursor`, requiring exactly `arity` string
10
+ * parts. `surface` names the caller for the error. */
11
+ export declare function decodeCursor(cursor: string, surface: string, arity: number): string[];
@@ -0,0 +1,35 @@
1
+ // Keyset cursors for every paged list surface (mcp-api §1: "truncated + cursor").
2
+ // One encoding — base64url of a JSON tuple of strings — shared by the OQX
3
+ // collect page (`[path, id]`) and the docs_list / docs_tree pages (`[path]`),
4
+ // so there is exactly one place a cursor can be malformed and one error for it.
5
+ // Kernel-owned: the query layer and the read layer both consume this; neither
6
+ // defines its own.
7
+ /** A cursor this surface did not issue (malformed / tampered / from another
8
+ * tool). The MCP layer maps it to `filter_invalid`. */
9
+ export class CursorInvalid extends Error {
10
+ surface;
11
+ constructor(surface) {
12
+ super("invalid cursor");
13
+ this.surface = surface;
14
+ this.name = "CursorInvalid";
15
+ }
16
+ }
17
+ /** Encode a keyset position as an opaque cursor. */
18
+ export function encodeCursor(parts) {
19
+ return Buffer.from(JSON.stringify(parts), "utf8").toString("base64url");
20
+ }
21
+ /** Decode a cursor issued by `encodeCursor`, requiring exactly `arity` string
22
+ * parts. `surface` names the caller for the error. */
23
+ export function decodeCursor(cursor, surface, arity) {
24
+ let parts;
25
+ try {
26
+ parts = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
27
+ }
28
+ catch {
29
+ throw new CursorInvalid(surface);
30
+ }
31
+ if (!Array.isArray(parts) || parts.length !== arity || !parts.every((p) => typeof p === "string"))
32
+ throw new CursorInvalid(surface);
33
+ return parts;
34
+ }
35
+ //# sourceMappingURL=cursor.js.map
@@ -0,0 +1,23 @@
1
+ export declare function sha256(input: string | Uint8Array): Buffer;
2
+ export declare function hashHex(input: string | Uint8Array): string;
3
+ /** Display form: first 16 hex chars (02 §1). */
4
+ export declare function shortHash(hash: Buffer | string): string;
5
+ /** raw_hash — sha256 of the exact raw source bytes (02 §1). */
6
+ export declare function rawHash(raw: string): Buffer;
7
+ export declare function normalizeText(raw: string): string;
8
+ export declare function normalizeVisibleText(raw: string, type: string): string;
9
+ /** norm_hash — sha256 of the normalized text (02 §1, §5.2). */
10
+ export declare function normHash(normalizedText: string): Buffer;
11
+ export declare function canonicalAttrs(attrs: Record<string, unknown>): string;
12
+ export interface TreeEntry {
13
+ blockId: string;
14
+ rawHashHex: string;
15
+ childTreeHashHex: string | null;
16
+ type: string;
17
+ attrs: Record<string, unknown>;
18
+ triviaHashHex: string | null;
19
+ }
20
+ /** Canonical serialization of a tree node's entries (UTF-8, no whitespace). */
21
+ export declare function serializeTreeEntries(entries: TreeEntry[]): string;
22
+ /** Tree-node hash: sha256 over the canonical entries serialization (02 §5.1). */
23
+ export declare function treeHash(entries: TreeEntry[]): Buffer;
@@ -0,0 +1,82 @@
1
+ import { createHash } from "node:crypto";
2
+ // Hashing & canonical serialization — 02 §5.
3
+ // Content hashes are sha256 of UTF-8 bytes, stored as 32-byte buffers and
4
+ // displayed truncated to 16 hex chars.
5
+ export function sha256(input) {
6
+ const h = createHash("sha256");
7
+ h.update(typeof input === "string" ? Buffer.from(input, "utf8") : input);
8
+ return h.digest();
9
+ }
10
+ export function hashHex(input) {
11
+ return sha256(input).toString("hex");
12
+ }
13
+ /** Display form: first 16 hex chars (02 §1). */
14
+ export function shortHash(hash) {
15
+ const hex = typeof hash === "string" ? hash : hash.toString("hex");
16
+ return hex.slice(0, 16);
17
+ }
18
+ /** raw_hash — sha256 of the exact raw source bytes (02 §1). */
19
+ export function rawHash(raw) {
20
+ return sha256(raw);
21
+ }
22
+ // Normalized text (02 §5.2): per-line trim + internal whitespace-run collapse,
23
+ // drop blank lines, NFC. Inline Markdown characters are preserved as content.
24
+ export function normalizeText(raw) {
25
+ return raw
26
+ .split(/\r\n|\r|\n/)
27
+ .map((line) => line.trim().replace(/[ \t]+/g, " "))
28
+ .filter((line) => line.length > 0)
29
+ .join(" ")
30
+ .normalize("NFC");
31
+ }
32
+ // Type-aware visible text (02 §5.2). Strips block-level syntax markers that are
33
+ // not content: ATX heading hashes and list/task markers. Inline markers stay.
34
+ export function normalizeVisibleText(raw, type) {
35
+ let s = raw;
36
+ if (type === "heading") {
37
+ s = s.replace(/^[ \t]*#{1,6}[ \t]+/, "").replace(/[ \t]+#*[ \t]*$/, "");
38
+ }
39
+ else if (type === "list_item" || type === "task") {
40
+ // Strip the list marker and an optional GFM checkbox on the first line.
41
+ s = s.replace(/^[ \t]*([-*+]|\d+[.)])[ \t]+/, "").replace(/^\[[ xX]\][ \t]+/, "");
42
+ }
43
+ return normalizeText(s);
44
+ }
45
+ /** norm_hash — sha256 of the normalized text (02 §1, §5.2). */
46
+ export function normHash(normalizedText) {
47
+ return sha256(normalizedText);
48
+ }
49
+ // Canonical attrs JSON: object with lexicographically sorted keys, no
50
+ // whitespace (02 §5.1). Values are emitted with JSON.stringify (stable for
51
+ // scalars/arrays/nested objects we use).
52
+ export function canonicalAttrs(attrs) {
53
+ const keys = Object.keys(attrs).sort();
54
+ const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalValue(attrs[k])}`);
55
+ return `{${parts.join(",")}}`;
56
+ }
57
+ function canonicalValue(v) {
58
+ if (v === null || typeof v !== "object")
59
+ return JSON.stringify(v);
60
+ if (Array.isArray(v))
61
+ return `[${v.map(canonicalValue).join(",")}]`;
62
+ const obj = v;
63
+ const keys = Object.keys(obj).sort();
64
+ return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalValue(obj[k])}`).join(",")}}`;
65
+ }
66
+ /** Canonical serialization of a tree node's entries (UTF-8, no whitespace). */
67
+ export function serializeTreeEntries(entries) {
68
+ const rows = entries.map((e) => [
69
+ JSON.stringify(e.blockId),
70
+ JSON.stringify(e.rawHashHex),
71
+ e.childTreeHashHex === null ? "null" : JSON.stringify(e.childTreeHashHex),
72
+ JSON.stringify(e.type),
73
+ canonicalAttrs(e.attrs),
74
+ e.triviaHashHex === null ? "null" : JSON.stringify(e.triviaHashHex),
75
+ ].join(","));
76
+ return `[${rows.map((r) => `[${r}]`).join(",")}]`;
77
+ }
78
+ /** Tree-node hash: sha256 over the canonical entries serialization (02 §5.1). */
79
+ export function treeHash(entries) {
80
+ return sha256(serializeTreeEntries(entries));
81
+ }
82
+ //# sourceMappingURL=hash.js.map
@@ -0,0 +1,6 @@
1
+ export type IdPrefix = "d" | "b" | "c" | "r" | "x" | "col" | "cp" | "e" | "rp" | "v" | "src";
2
+ export declare function randomSuffix(len?: number): string;
3
+ /** Mint an ID with the given prefix. Collision checking is the store's job. */
4
+ export declare function mintId(prefix: IdPrefix): string;
5
+ export declare function isValidId(id: string, prefix?: IdPrefix): boolean;
6
+ export declare function prefixOf(id: string): string | null;
@@ -0,0 +1,26 @@
1
+ import { randomInt } from "node:crypto";
2
+ // Crockford base32 alphabet, lowercased (no i, l, o, u).
3
+ const ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz";
4
+ const ID_LEN = 7;
5
+ export function randomSuffix(len = ID_LEN) {
6
+ let s = "";
7
+ for (let i = 0; i < len; i++)
8
+ s += ALPHABET[randomInt(ALPHABET.length)];
9
+ return s;
10
+ }
11
+ /** Mint an ID with the given prefix. Collision checking is the store's job. */
12
+ export function mintId(prefix) {
13
+ return `${prefix}_${randomSuffix()}`;
14
+ }
15
+ const ID_RE = new RegExp(`^([a-z]+)_([${ALPHABET}]{${ID_LEN}})$`);
16
+ export function isValidId(id, prefix) {
17
+ const m = ID_RE.exec(id);
18
+ if (!m)
19
+ return false;
20
+ return prefix === undefined || m[1] === prefix;
21
+ }
22
+ export function prefixOf(id) {
23
+ const m = ID_RE.exec(id);
24
+ return m ? m[1] : null;
25
+ }
26
+ //# sourceMappingURL=ids.js.map
@@ -0,0 +1 @@
1
+ export declare const VERSION = "0.1.0";
@@ -0,0 +1,2 @@
1
+ export const VERSION = "0.1.0";
2
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,43 @@
1
+ import { Store } from "./store/store.js";
2
+ import { type TreeInputBlock } from "./store/writers.js";
3
+ import type { RawBlock } from "./parse/types.js";
4
+ export interface DispositionRow {
5
+ blockId: string;
6
+ kind: string;
7
+ confidence: number | null;
8
+ reason: string | null;
9
+ matcherV: string | null;
10
+ detail: Record<string, unknown>;
11
+ }
12
+ export interface ResolvedEdgeRow {
13
+ srcDoc: string;
14
+ srcBlock: string | null;
15
+ srcField: string | null;
16
+ predicate: string;
17
+ dstKind: "document" | "block" | "external" | "collection";
18
+ dstNode: string;
19
+ anchor: string | null;
20
+ provenance: string;
21
+ }
22
+ export type IdResolver = (rest: RawBlock[], docId: string | null) => {
23
+ assigned: TreeInputBlock[];
24
+ dispositions: DispositionRow[];
25
+ deleted: string[];
26
+ consumedPool?: string[];
27
+ extractEdges?: (docId: string, frontmatter: Record<string, unknown>) => ResolvedEdgeRow[];
28
+ };
29
+ export interface IngestResult {
30
+ docId: string;
31
+ commitId: string;
32
+ revId: string;
33
+ blockCount: number;
34
+ /** convergence: sha256(file bytes) === revision.rendered_hash */
35
+ converged: boolean;
36
+ }
37
+ /** Ingest a single file's content into the store as one observed commit. */
38
+ export declare function ingestFile(store: Store, repoId: string, path: string, content: string, opts?: {
39
+ ts?: string;
40
+ origin?: "observed" | "import";
41
+ resolveIds?: IdResolver;
42
+ format?: string;
43
+ }): IngestResult;
@@ -0,0 +1,287 @@
1
+ import { Store } from "./store/store.js";
2
+ import { parseTree } from "./parse/tree.js";
3
+ import { render } from "./parse/render.js";
4
+ import { reconstructContent } from "./read/document.js";
5
+ import { assignIds, writeBlockTree, putBlob, newCommit, writeRevision } from "./store/writers.js";
6
+ import { sha256, normalizeVisibleText } from "./hash.js";
7
+ import { mintId } from "./ids.js";
8
+ import { keyBetween } from "./order-key.js";
9
+ import { ftsDeleteDoc, ftsIndexDoc } from "./store/fts.js";
10
+ import { rebuildSections } from "./store/sections.js";
11
+ import { writeDocNodes, projectSectionNodes } from "./store/nodes.js";
12
+ import { flattenFrontmatter, flattenComputed, writeDocProperties } from "./store/properties.js";
13
+ import { maintainEdges, adoptPhantoms } from "./store/edges.js";
14
+ import { parse as parseYaml } from "yaml";
15
+ import { adapterForPath } from "../format/index.js";
16
+ function extractFrontmatter(blocks) {
17
+ if (blocks.length > 0 && blocks[0].type === "frontmatter") {
18
+ return { fmBlock: blocks[0], rest: blocks.slice(1) };
19
+ }
20
+ return { fmBlock: null, rest: blocks };
21
+ }
22
+ // Parse a frontmatter block's raw (incl. --- fences) into a JSON-safe object.
23
+ // Malformed YAML yields {} — frontmatter is queryable metadata, not load-bearing.
24
+ function parseFrontmatter(fmBlock) {
25
+ if (!fmBlock)
26
+ return {};
27
+ const body = fmBlock.raw.replace(/^---\r?\n/, "").replace(/\r?\n?---\s*$/, "");
28
+ try {
29
+ const parsed = parseYaml(body);
30
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed)
31
+ ? parsed
32
+ : {};
33
+ }
34
+ catch {
35
+ return {};
36
+ }
37
+ }
38
+ function flatten(blocks, parent, depth, ancestorPath, out) {
39
+ let prevKey = null;
40
+ blocks.forEach((b, ordinal) => {
41
+ const orderKey = keyBetween(prevKey, null);
42
+ prevKey = orderKey;
43
+ const visible = normalizeVisibleText(b.raw, b.type);
44
+ out.push({
45
+ blockId: b.blockId,
46
+ parentBlock: parent,
47
+ orderKey,
48
+ ordinal,
49
+ depth,
50
+ ancestorPath,
51
+ type: b.type,
52
+ attrs: JSON.stringify(b.attrs),
53
+ text: visible,
54
+ rawHash: sha256(b.raw),
55
+ normHash: sha256(visible),
56
+ triviaHash: b.trivia.length > 0 ? sha256(b.trivia) : null,
57
+ });
58
+ if (b.children.length > 0) {
59
+ flatten(b.children, b.blockId, depth + 1, `${ancestorPath}${b.blockId}/`, out);
60
+ }
61
+ });
62
+ }
63
+ /** Ingest a single file's content into the store as one observed commit. */
64
+ export function ingestFile(store, repoId, path, content, opts = {}) {
65
+ const ts = opts.ts ?? new Date().toISOString();
66
+ const origin = opts.origin ?? "observed";
67
+ const adapter = adapterForPath(path);
68
+ const format = opts.format ?? adapter?.format ?? "markdown";
69
+ return store.write((db) => {
70
+ const tree = adapter ? adapter.parse(content) : parseTree(content);
71
+ const { fmBlock, rest } = extractFrontmatter(tree.children);
72
+ // Frontmatter blob (markdown-specific) preserved for revision history.
73
+ const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
74
+ // The exact separator bytes between the frontmatter block and the first body
75
+ // block (the frontmatter block's trailing trivia). Persisted so the body
76
+ // reconstructs byte-for-byte instead of assuming a canonical blank line.
77
+ // NULL when there is no frontmatter.
78
+ const fmTrivia = fmBlock ? fmBlock.trivia : null;
79
+ // Document property bag: adapter-provided for non-markdown formats,
80
+ // frontmatter-parsed for markdown. Flattened into the properties table
81
+ // below (source=frontmatter); also drives frontmatter edge extraction.
82
+ const metadata = (adapter?.extractMetadata)
83
+ ? (adapter.extractMetadata(content) ?? parseFrontmatter(fmBlock))
84
+ : parseFrontmatter(fmBlock);
85
+ // Upsert the document row.
86
+ let doc = db.prepare("SELECT doc_id FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
87
+ const docId = doc?.doc_id ?? mintId("d");
88
+ const isNew = !doc;
89
+ if (!doc) {
90
+ db.prepare("INSERT INTO docs (doc_id, repo_id, path, format, leading_trivia, frontmatter_trivia) VALUES (?, ?, ?, ?, ?, ?)").run(docId, repoId, path, format, tree.leadingTrivia, fmTrivia);
91
+ doc = { doc_id: docId };
92
+ // Adopt phantom edges that pointed at this path so backlinks re-point.
93
+ adoptPhantoms(db, path, docId);
94
+ }
95
+ else {
96
+ db.prepare("UPDATE docs SET format = ?, leading_trivia = ?, frontmatter_trivia = ? WHERE doc_id = ?").run(format, tree.leadingTrivia, fmTrivia, docId);
97
+ }
98
+ // Assign block ids via the resolver when supplied (it reconciles against
99
+ // the prior revision — for a new doc that's an empty old tree, so all blocks
100
+ // mint — and also drives edge extraction). Without a resolver, fresh-mint.
101
+ const resolved = opts.resolveIds
102
+ ? opts.resolveIds(rest, isNew ? null : docId)
103
+ : { assigned: assignIds(rest), dispositions: [], deleted: [] };
104
+ const assigned = resolved.assigned;
105
+ const rootTreeHex = writeBlockTree(db, assigned);
106
+ const commit = newCommit(db, { repoId, ts, origin });
107
+ const renderedHash = sha256(content);
108
+ const rev = writeRevision(db, {
109
+ docId,
110
+ rootTreeHex,
111
+ frontmatterBlobHex: fmBlobHex,
112
+ renderedHash,
113
+ path,
114
+ commitId: commit.commitId,
115
+ });
116
+ // Snapshot deleted blocks into the resurrection pool BEFORE dropping the
117
+ // doc's rows (the ingest rebuilds current-state blocks wholesale).
118
+ if (resolved.deleted.length > 0) {
119
+ const expires = new Date(Date.parse(ts) + 30 * 24 * 3600 * 1000).toISOString();
120
+ const pool = db.prepare(`INSERT OR REPLACE INTO resurrection_pool (block_id, repo_id, doc_id, raw_hash, norm_hash, type, deleted_commit, expires_ts)
121
+ SELECT block_id, repo_id, doc_id, raw_hash, norm_hash, type, ?, ? FROM blocks WHERE block_id = ? AND doc_id = ?`);
122
+ for (const id of resolved.deleted)
123
+ pool.run(commit.commitId, expires, id, docId);
124
+ }
125
+ // Refresh current-state blocks (clear + repopulate with carried/minted ids).
126
+ // FTS is external-content: delete old index rows before dropping blocks.
127
+ ftsDeleteDoc(db, docId);
128
+ db.prepare("DELETE FROM blocks WHERE doc_id = ?").run(docId);
129
+ const rows = [];
130
+ flatten(assigned, null, 0, "/", rows);
131
+ const insert = db.prepare(`INSERT INTO blocks
132
+ (block_id, repo_id, doc_id, parent_block, order_key, ordinal, depth,
133
+ ancestor_path, type, attrs, text, raw_hash, norm_hash, trivia_hash, created_commit)
134
+ VALUES
135
+ (@blockId, @repoId, @docId, @parentBlock, @orderKey, @ordinal, @depth,
136
+ @ancestorPath, @type, @attrs, @text, @rawHash, @normHash, @triviaHash, @createdCommit)`);
137
+ for (const r of rows) {
138
+ insert.run({ ...r, repoId, docId, createdCommit: commit.commitId });
139
+ }
140
+ ftsIndexDoc(db, docId);
141
+ rebuildSections(db, docId);
142
+ // Node projection: adapter-provided semantic features from parsed blocks,
143
+ // plus engine-derived `md:section` nodes from the (just-rebuilt) sections
144
+ // table. Written together in one pass (writeDocNodes clears + repopulates).
145
+ const propertyRows = [];
146
+ const sectionNodes = projectSectionNodes(db, docId);
147
+ if (adapter?.projectNodes) {
148
+ // Zip assigned block ids onto the parsed blocks so projectNodes can anchor
149
+ // each node to its own block (the parsed RawBlocks are pre-identity).
150
+ const rawBlocks = zipAssignedIds(rest, assigned);
151
+ const projected = adapter.projectNodes(rawBlocks);
152
+ // Keep only nodes anchored to a real assigned block id.
153
+ const idSet = collectBlockIds(assigned);
154
+ const withIds = projected.map((n) => ({
155
+ ...n,
156
+ blockId: n.blockId && idSet.has(n.blockId) ? n.blockId : "",
157
+ }));
158
+ writeDocNodes(db, repoId, docId, [...withIds, ...sectionNodes]);
159
+ // Inline properties (key:: value) → property rows (source=inline). The
160
+ // authored shape within the inline source drives `card`: a key that occurs
161
+ // exactly once in the document is `scalar` (so `element == "fire"` can
162
+ // match a lone `element:: fire`); a key that repeats is `list`. Repeats
163
+ // accumulate as ord-indexed rows in document order (block ordinal, then
164
+ // in-block position — the order withIds already carries). Effective bare-
165
+ // key cardinality (a frontmatter+inline collision) is decided at query
166
+ // time by the single-value gate on scalar comparisons (see compile.ts),
167
+ // not here — each source stays honest to its own authored shape.
168
+ const inlineFields = withIds.filter((n) => n.kind === "md:inline_field" && n.name);
169
+ const inlineCounts = new Map();
170
+ for (const n of inlineFields)
171
+ inlineCounts.set(n.name, (inlineCounts.get(n.name) ?? 0) + 1);
172
+ const inlineOrd = new Map();
173
+ for (const n of inlineFields) {
174
+ const ord = inlineOrd.get(n.name) ?? 0;
175
+ inlineOrd.set(n.name, ord + 1);
176
+ propertyRows.push({
177
+ source: "inline",
178
+ blockId: n.blockId || null,
179
+ key: n.name,
180
+ card: (inlineCounts.get(n.name) ?? 0) > 1 ? "list" : "scalar",
181
+ ord,
182
+ ...typedInlineValue(n.value),
183
+ });
184
+ }
185
+ }
186
+ else if (sectionNodes.length > 0) {
187
+ // No adapter node projection, but the document still has sections.
188
+ writeDocNodes(db, repoId, docId, sectionNodes);
189
+ }
190
+ // Frontmatter → property rows (source=frontmatter). Flatten nested maps to
191
+ // dotted keys, arrays to ord-indexed list rows, scalars to a scalar row.
192
+ for (const r of flattenFrontmatter(metadata)) {
193
+ propertyRows.push({ source: "frontmatter", blockId: null, ...r });
194
+ }
195
+ // Computed → property rows (source=computed). Engine-derived $-intrinsics
196
+ // ($title, $tags) from the adapter; never claim authored keys.
197
+ if (adapter?.computeProperties) {
198
+ const computed = adapter.computeProperties(rest, metadata);
199
+ for (const r of flattenComputed(computed)) {
200
+ propertyRows.push({ source: "computed", blockId: null, ...r });
201
+ }
202
+ }
203
+ writeDocProperties(db, repoId, docId, commit.commitId, propertyRows);
204
+ // Edge extraction + interval maintenance (05 §2), if the resolver supplies
205
+ // an extractor. Runs in this commit transaction.
206
+ if (resolved.extractEdges) {
207
+ const extracted = resolved.extractEdges(docId, metadata);
208
+ maintainEdges(db, repoId, docId, commit.commitId, extracted);
209
+ }
210
+ // Persist dispositions + block-history projection. With a resolver, use the
211
+ // real reconciliation dispositions; otherwise record every block as inserted.
212
+ const dispositions = resolved.dispositions.length > 0 || opts.resolveIds
213
+ ? resolved.dispositions
214
+ : rows.map((r) => ({ blockId: r.blockId, kind: "inserted", confidence: null, reason: null, matcherV: null, detail: {} }));
215
+ const insDisp = db.prepare(`INSERT OR IGNORE INTO dispositions (commit_id, block_id, kind, confidence, reason, matcher_v, detail)
216
+ VALUES (?, ?, ?, ?, ?, ?, ?)`);
217
+ const bc = db.prepare("INSERT OR IGNORE INTO block_changes (block_id, commit_id, kind) VALUES (?, ?, ?)");
218
+ for (const d of dispositions) {
219
+ insDisp.run(commit.commitId, d.blockId, d.kind, d.confidence, d.reason, d.matcherV, JSON.stringify(d.detail));
220
+ bc.run(d.blockId, commit.commitId, d.kind);
221
+ }
222
+ // Consumed pool rows (resurrected) are removed.
223
+ for (const id of resolved.consumedPool ?? []) {
224
+ db.prepare("DELETE FROM resurrection_pool WHERE block_id = ?").run(id);
225
+ }
226
+ // Update document current pointers + convergence hash.
227
+ const fileHash = sha256(content);
228
+ db.prepare("UPDATE docs SET current_rev = ?, file_hash = ? WHERE doc_id = ?").run(rev.revId, fileHash, docId);
229
+ // Convergence check (01 §2): file bytes vs rendered revision. Two independent
230
+ // round-trips must both reproduce the source exactly:
231
+ // 1. the freshly-parsed in-memory tree (validates the parser), and
232
+ // 2. a reload from storage (validates that persisted blocks + trivia +
233
+ // frontmatter separator tile the source — the fidelity a plain read or
234
+ // an `apply` write actually depends on). Rendering only the in-memory
235
+ // tree hid store→reload separator loss; reconstructContent closes that.
236
+ const renderFn = adapter?.render ?? render;
237
+ const converged = fileHash.equals(renderedHash) &&
238
+ renderFn(tree) === content &&
239
+ reconstructContent(db, docId) === content;
240
+ return {
241
+ docId,
242
+ commitId: commit.commitId,
243
+ revId: rev.revId,
244
+ blockCount: rows.length,
245
+ converged,
246
+ };
247
+ });
248
+ }
249
+ // Zip assigned block ids onto the parsed RawBlocks for node projection. The
250
+ // parsed `rest` and `assigned` trees are the same blocks in the same order and
251
+ // nesting (assigned is a 1:1 id-carrying projection of rest), so we walk them in
252
+ // lockstep and stamp each RawBlock's blockId. Returns fresh RawBlocks (originals
253
+ // untouched — blockId is a projection-only field).
254
+ function zipAssignedIds(parsed, assigned) {
255
+ return parsed.map((b, i) => {
256
+ const a = assigned[i];
257
+ return {
258
+ ...b,
259
+ ...(a ? { blockId: a.blockId } : {}),
260
+ children: a ? zipAssignedIds(b.children, a.children) : b.children,
261
+ };
262
+ });
263
+ }
264
+ // Inline field values arrive as captured strings; coerce to a typed column so
265
+ // numeric/bool inline props (priority:: 3) compare like their frontmatter kin.
266
+ // A value that isn't cleanly numeric/bool stays a string.
267
+ function typedInlineValue(raw) {
268
+ const v = (raw ?? "").trim();
269
+ if (v === "true" || v === "false")
270
+ return { valText: null, valNum: null, valBool: v === "true" ? 1 : 0, valJson: null, type: "bool" };
271
+ if (v !== "" && Number.isFinite(Number(v)))
272
+ return { valText: null, valNum: Number(v), valBool: null, valJson: null, type: "number" };
273
+ return { valText: v, valNum: null, valBool: null, valJson: null, type: "string" };
274
+ }
275
+ function collectBlockIds(blocks) {
276
+ const ids = new Set();
277
+ const walk = (list) => {
278
+ for (const b of list) {
279
+ ids.add(b.blockId);
280
+ if (b.children.length > 0)
281
+ walk(b.children);
282
+ }
283
+ };
284
+ walk(blocks);
285
+ return ids;
286
+ }
287
+ //# sourceMappingURL=ingest.js.map
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Return a key strictly between a and b (exclusive). null bounds mean
3
+ * open-ended (a=null → before b; b=null → after a). Amortized O(1) growth for
4
+ * sequential appends.
5
+ */
6
+ export declare function keyBetween(a: string | null, b: string | null): string;
7
+ /** Generate n sequential append keys starting from scratch. */
8
+ export declare function sequentialKeys(n: number): string[];