@amoree-code/ocean 0.0.0-stage → 1.2.0

package/node_modules/@ocean/brain/dist/brain/brain-eval.js

@@ -0,0 +1,29 @@
+import { brainSearch } from "./brain-service.js";
+export async function evaluateRecall(cases, options) {
+    const k = options.k ?? 5;
+    const misses = [];
+    for (const item of cases) {
+        const expect = [item.expect].flat();
+        const { results } = await brainSearch({
+            query: item.question,
+            limit: k,
+            root: options.root,
+            embedder: options.embedder ?? null,
+            indexPort: options.indexPort,
+            scope: item.project === undefined ? undefined : { project: item.project },
+        });
+        const got = results.map((hit) => hit.file);
+        const found = results.some((hit) => expect.includes(hit.path) || expect.includes(hit.file));
+        if (!found)
+            misses.push({ question: item.question, expect, got });
+    }
+    const hits = cases.length - misses.length;
+    return {
+        k,
+        total: cases.length,
+        hits,
+        recall: cases.length ? hits / cases.length : 0,
+        misses,
+    };
+}
+//# sourceMappingURL=brain-eval.js.map

package/node_modules/@ocean/brain/dist/brain/brain-markdown.js

@@ -0,0 +1,226 @@
+import path from "node:path";
+import { validateBrainRecord } from "@ocean/core/domain/brain/brain-record.js";
+import { normalizeScope } from "@ocean/core/domain/brain/record-scope.js";
+import { parseFrontmatter } from "@ocean/core/shared/fs-utils.js";
+// Pure markdown layer for T-228: turns one markdown file's raw text into a canonical
+// BrainRecord plus a per-field conformance report, extracts its wikilinks, and chunks its
+// body. No I/O here — the caller (brain-reindex.ts) reads the file and passes the text in.
+function splitFrontmatter(source) {
+    if (!source.startsWith("---"))
+        return { block: "", body: source };
+    const end = source.indexOf("\n---", 3);
+    if (end < 0)
+        return { block: source.slice(3), body: "" };
+    const afterMarker = source.indexOf("\n", end + 1);
+    return {
+        block: source.slice(3, end),
+        body: afterMarker < 0 ? "" : source.slice(afterMarker + 1),
+    };
+}
+// Tags can be declared inline (`tags: [a, b]`), as a block list, or omitted entirely.
+function extractTags(block) {
+    const lines = block.split("\n");
+    const inlineIndex = lines.findIndex((line) => /^\s{0,2}tags:\s*\[/.test(line));
+    if (inlineIndex >= 0) {
+        const match = /\[(.*)\]/.exec(lines[inlineIndex]);
+        if (!match)
+            return [];
+        return match[1]
+            .split(",")
+            .map((tag) => tag.trim().replace(/^["']|["']$/g, ""))
+            .filter(Boolean);
+    }
+    const headerIndex = lines.findIndex((line) => /^\s{0,2}tags:\s*$/.test(line));
+    if (headerIndex < 0)
+        return [];
+    const tags = [];
+    for (let index = headerIndex + 1; index < lines.length; index += 1) {
+        const match = /^\s+-\s*(.+)$/.exec(lines[index]);
+        if (!match)
+            break;
+        tags.push(match[1].trim().replace(/^["']|["']$/g, ""));
+    }
+    return tags;
+}
+function firstHeading(body) {
+    const match = /^#\s+(.+)$/m.exec(body);
+    return match ? match[1].trim() : null;
+}
+// relPath is the record's identity of last resort: `<store-relative path, no .md>`.
+export function normalizeRecord(relPath, rawSource) {
+    // Normalized up front (see fs-utils.ts parseFrontmatter): splitFrontmatter and chunkBody
+    // both match line regexes against `$`, which a CRLF trailing `\r` (e.g. a Windows git
+    // checkout) defeats since `.` excludes line terminators.
+    const source = rawSource.replace(/\r\n/g, "\n");
+    const { block, body } = splitFrontmatter(source);
+    const fields = parseFrontmatter(source);
+    const fallbackId = relPath.replace(/\.md$/i, "");
+    const declaredOrAliased = (canonical, ...aliases) => {
+        if (fields[canonical] !== undefined)
+            return { value: fields[canonical], state: "declared" };
+        for (const alias of aliases) {
+            if (fields[alias] !== undefined)
+                return { value: fields[alias], state: "aliased" };
+        }
+        return { value: undefined, state: "defaulted" };
+    };
+    const idField = declaredOrAliased("id", "name");
+    const id = idField.value ?? fallbackId;
+    const idState = idField.value ? idField.state : "defaulted";
+    const titleField = declaredOrAliased("title", "name");
+    const heading = firstHeading(body);
+    const title = titleField.value ?? heading ?? fallbackId;
+    const titleState = titleField.value
+        ? titleField.state
+        : heading
+            ? "aliased"
+            : "defaulted";
+    const summaryField = declaredOrAliased("summary", "description");
+    const summary = summaryField.value ?? "";
+    const tags = extractTags(block);
+    const tagsState = tags.length > 0 ? "declared" : "defaulted";
+    const typeField = declaredOrAliased("type");
+    const confidenceField = declaredOrAliased("confidence");
+    const createdField = declaredOrAliased("created");
+    const updatedField = declaredOrAliased("updated");
+    const lastConfirmedField = declaredOrAliased("last_confirmed_at", "last_verified");
+    const scopeField = declaredOrAliased("scope");
+    // A project memory record (project/<id>/…, T-296) or project doc (project-docs/<id>/…, T-307)
+    // without a declared scope belongs to its project, never to every project.
+    const owningProject = /^project(?:-docs)?\/([^/]+)\//.exec(relPath)?.[1];
+    const scope = normalizeScope(scopeField.value ??
+        (owningProject ? `project:${owningProject}` : undefined));
+    const record = validateBrainRecord({
+        id,
+        title,
+        summary,
+        tags,
+        type: typeField.value ?? "unknown",
+        confidence: confidenceField.value ?? "unknown",
+        created: createdField.value ?? null,
+        updated: updatedField.value ?? null,
+        lastConfirmedAt: lastConfirmedField.value ?? null,
+        scope: scope.scope,
+    });
+    const conformance = {
+        id: idState,
+        title: titleState,
+        summary: summaryField.state,
+        tags: tagsState,
+        type: typeField.state,
+        confidence: confidenceField.state,
+        created: createdField.state,
+        updated: updatedField.state,
+        lastConfirmedAt: lastConfirmedField.state,
+        // An invalid scope is counted by reindex's invalidScopes, not as a conformance state.
+        scope: scopeField.value === undefined || scope.state === "defaulted"
+            ? "defaulted"
+            : "declared",
+    };
+    return { record, conformance, body, rawScope: scopeField.value };
+}
+// Extracts `[[target]]`, `[[target|alias]]`, `[[target#heading]]`, ignoring fenced code
+// blocks (``` ... ```) so a documentation example is never mistaken for a real link.
+export function extractWikilinks(body) {
+    const withoutCode = body.replace(/```[\s\S]*?```/g, "");
+    const links = [];
+    const pattern = /\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]/g;
+    let match;
+    // biome-ignore lint/suspicious/noAssignInExpressions: standard regex-exec loop
+    while ((match = pattern.exec(withoutCode)) !== null) {
+        links.push({
+            target: match[1].trim(),
+            heading: match[2]?.trim() ?? null,
+            alias: match[3]?.trim() ?? null,
+        });
+    }
+    return links;
+}
+const MAX_CHUNK_CHARS = 2_000;
+// Splits on `##` headings (level-2+ only; a level-1 `#` title stays attached to the chunk
+// that follows it), then further splits any resulting section longer than MAX_CHUNK_CHARS at
+// a paragraph boundary. Ordinals are stable and start at 0.
+export function chunkBody(body) {
+    const sections = [];
+    const lines = body.split("\n");
+    let currentHeading = null;
+    let currentLines = [];
+    const flush = () => {
+        const text = currentLines.join("\n").trim();
+        if (text)
+            sections.push({ heading: currentHeading, text });
+        currentLines = [];
+    };
+    for (const line of lines) {
+        const heading = /^##\s+(.+)$/.exec(line);
+        if (heading) {
+            flush();
+            currentHeading = heading[1].trim();
+            continue;
+        }
+        currentLines.push(line);
+    }
+    flush();
+    if (sections.length === 0)
+        return [];
+    const chunks = [];
+    let ordinal = 0;
+    for (const section of sections) {
+        if (section.text.length <= MAX_CHUNK_CHARS) {
+            chunks.push({
+                ordinal: ordinal++,
+                heading: section.heading,
+                text: section.text,
+            });
+            continue;
+        }
+        const paragraphs = section.text.split(/\n\n+/);
+        let buffer = "";
+        for (const paragraph of paragraphs) {
+            const candidate = buffer ? `${buffer}\n\n${paragraph}` : paragraph;
+            if (candidate.length > MAX_CHUNK_CHARS && buffer) {
+                chunks.push({
+                    ordinal: ordinal++,
+                    heading: section.heading,
+                    text: buffer,
+                });
+                buffer = paragraph;
+            }
+            else {
+                buffer = candidate;
+            }
+        }
+        if (buffer)
+            chunks.push({
+                ordinal: ordinal++,
+                heading: section.heading,
+                text: buffer,
+            });
+    }
+    return chunks;
+}
+// Arabic harakat (U+064B-U+065F, U+0670) and tatweel (U+0640) are stripped so a diacritized
+// word matches its bare form under FTS5's unicode61 tokenizer (which itself only removes
+// diacritics from the *default* Unicode diacritics list, and only when told to — this
+// normalization is what actually makes indexing and querying agree). أ/إ/آ fold to ا and ي
+// folds to ی(kmr) — wait: folds ك to ک is a Sorani/Kurmanji orthography convenience. ة and ه
+// (Sorani) are intentionally left alone: folding would collide with distinct Sorani/Kurmanji
+// words. Applied identically at index time and query time so the tokenizer sees the same
+// bytes both times.
+const HARAKAT_AND_TATWEEL = /[ً-ٰٟـ]/g;
+export function normalizeForSearch(text) {
+    return text
+        .normalize("NFKC")
+        .replace(HARAKAT_AND_TATWEEL, "")
+        .replace(/[أإآ]/g, "ا")
+        .replace(/ي/g, "ی")
+        .replace(/ك/g, "ک")
+        .toLowerCase();
+}
+export function idFromRelPath(relPath) {
+    return relPath.replace(/\.md$/i, "");
+}
+export function basenameStem(relPath) {
+    return path.basename(relPath, path.extname(relPath));
+}
+//# sourceMappingURL=brain-markdown.js.map

package/node_modules/@ocean/brain/dist/brain/brain-reindex.js

@@ -0,0 +1,247 @@
+import { createHash } from "node:crypto";
+import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { INDEX_DIR, KNOWLEDGE_DIR, listProjectDirs, oceanRoot, resolveWithin, STORE_DIR, } from "@ocean/core/shared/paths.js";
+import { basenameStem, chunkBody, extractWikilinks, normalizeForSearch, normalizeRecord, } from "./brain-markdown.js";
+const STORES = ["memory", "knowledge"];
+const MAX_FILES = 10_000;
+async function walk(root, depth = 0) {
+    if (depth === 0) {
+        try {
+            await stat(root);
+        }
+        catch {
+            return [];
+        }
+    }
+    let entries;
+    try {
+        entries = await readdir(root, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    const files = [];
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+        if (entry.name.startsWith("."))
+            continue;
+        if (entry.isDirectory()) {
+            if (entry.name === "archive")
+                continue;
+            files.push(...(await walk(path.join(root, entry.name), depth + 1)));
+            continue;
+        }
+        if (entry.name.endsWith(".md"))
+            files.push(path.join(root, entry.name));
+    }
+    return files;
+}
+async function corpusFiles(root) {
+    const found = [];
+    const add = async (prefix, storeRoot) => {
+        for (const file of await walk(storeRoot))
+            found.push({
+                prefix,
+                storePath: `${prefix}/${path.relative(storeRoot, file).split(path.sep).join("/")}`,
+                file,
+            });
+    };
+    for (const store of STORES)
+        await add(store, resolveWithin(root, STORE_DIR[store]));
+    // listProjectDirs sorts by id; an id seen twice (two category folders) is indexed once.
+    const seen = new Set();
+    for (const { id, dir } of listProjectDirs(root)) {
+        if (seen.has(id))
+            continue;
+        seen.add(id);
+        await add(`project/${id}`, resolveWithin(root, dir, "memory"));
+        await add(`project-docs/${id}`, resolveWithin(root, dir, "docs"));
+    }
+    return found;
+}
+// Cheap staleness probe: hashes every indexed markdown file's bytes, in the same (sorted path,
+// sha256) shape reindexBrain folds into meta.corpus_hash, without parsing frontmatter or
+// building chunks. Used by brainSearch to report `stale: true` when the index is older than the
+// files it was built from, without paying for a full reindex on every search.
+export async function computeCorpusHash(root = oceanRoot()) {
+    const entries = [];
+    for (const { storePath, file } of await corpusFiles(root)) {
+        const source = await readFile(file, "utf8");
+        entries.push({
+            storePath,
+            contentHash: createHash("sha256").update(source).digest("hex"),
+        });
+    }
+    entries.sort((a, b) => a.storePath.localeCompare(b.storePath));
+    return createHash("sha256")
+        .update(entries
+        .map((entry) => `${entry.storePath}:${entry.contentHash}`)
+        .join("\n"))
+        .digest("hex");
+}
+// Resolves one wikilink target to a store-relative doc path, per the fixed precedence order:
+// exact declared id anywhere in the corpus, the same store's `<target>.md`, that store's
+// `<target>/README.md`, then a corpus-wide unique file stem. Anything else is dangling.
+// Exactly two or more stem matches is reported separately as ambiguous, not silently guessed.
+function resolveLink(targetRaw, sourceStore, byId, byPath, byStem) {
+    const cleaned = targetRaw.replace(/^\.\//, "");
+    const declaredId = byId.get(cleaned);
+    if (declaredId !== undefined)
+        return { resolved: declaredId, ambiguous: false };
+    const storeRelative = `${sourceStore}/${cleaned}.md`;
+    if (byPath.has(storeRelative))
+        return { resolved: storeRelative, ambiguous: false };
+    const readmePath = `${sourceStore}/${cleaned}/README.md`;
+    if (byPath.has(readmePath))
+        return { resolved: readmePath, ambiguous: false };
+    const stem = cleaned.split("/").pop() ?? cleaned;
+    const stemMatches = byStem.get(stem) ?? [];
+    if (stemMatches.length === 1)
+        return { resolved: stemMatches[0], ambiguous: false };
+    if (stemMatches.length > 1)
+        return { resolved: null, ambiguous: true };
+    return { resolved: null, ambiguous: false };
+}
+export async function reindexBrain(options) {
+    const root = options.root ?? oceanRoot();
+    const embedder = options.embedder ?? null;
+    const walked = [];
+    let totalSeen = 0;
+    for (const { prefix: store, storePath: relative, file } of await corpusFiles(root)) {
+        totalSeen += 1;
+        if (totalSeen > MAX_FILES)
+            throw new Error(`brain reindex found more than ${MAX_FILES} markdown files — refusing to index an unbounded corpus`);
+        const source = await readFile(file, "utf8");
+        const { record, conformance, body, rawScope } = normalizeRecord(relative, source);
+        const links = extractWikilinks(body).map((link) => ({
+            targetRaw: link.target,
+            heading: link.heading,
+            alias: link.alias,
+        }));
+        walked.push({
+            storePath: relative,
+            store,
+            record,
+            conformance,
+            rawScope,
+            body,
+            contentHash: createHash("sha256").update(source).digest("hex"),
+            bytes: Buffer.byteLength(source, "utf8"),
+            links,
+        });
+    }
+    walked.sort((a, b) => a.storePath.localeCompare(b.storePath));
+    // Duplicate *declared* ids fail loudly — a collision between two files whose ids only
+    // coincide because both defaulted to their own path can never happen (paths are unique).
+    const declaredIds = new Map();
+    for (const doc of walked) {
+        if (doc.conformance.id === "defaulted")
+            continue;
+        const existing = declaredIds.get(doc.record.id);
+        if (existing) {
+            throw new Error(`duplicate brain record id '${doc.record.id}' declared in both ${existing} and ${doc.storePath}`);
+        }
+        declaredIds.set(doc.record.id, doc.storePath);
+    }
+    const byId = new Map();
+    const byPath = new Map();
+    const byStem = new Map();
+    for (const doc of walked) {
+        byId.set(doc.record.id, doc.storePath);
+        byPath.set(doc.storePath, doc.storePath);
+        const stem = basenameStem(doc.storePath);
+        byStem.set(stem, [...(byStem.get(stem) ?? []), doc.storePath]);
+    }
+    const docInputs = [];
+    const linkInputs = [];
+    const danglingLinks = [];
+    const ambiguousLinks = [];
+    const conformanceTally = {};
+    const scopeTally = {};
+    const invalidScopes = [];
+    let chunkCount = 0;
+    for (const doc of walked) {
+        scopeTally[doc.record.scope] = (scopeTally[doc.record.scope] ?? 0) + 1;
+        if (doc.record.scope === "invalid")
+            invalidScopes.push({ path: doc.storePath, scope: doc.rawScope ?? "" });
+        for (const [field, state] of Object.entries(doc.conformance)) {
+            conformanceTally[field] ??= { aliased: 0, defaulted: 0 };
+            if (state === "aliased")
+                conformanceTally[field].aliased += 1;
+            if (state === "defaulted")
+                conformanceTally[field].defaulted += 1;
+        }
+        const chunks = chunkBody(doc.body);
+        chunkCount += chunks.length;
+        docInputs.push({
+            id: doc.record.id,
+            path: doc.storePath,
+            title: doc.record.title,
+            summary: doc.record.summary,
+            tags: doc.record.tags,
+            type: doc.record.type,
+            scope: doc.record.scope,
+            confidence: doc.record.confidence,
+            created: doc.record.created,
+            updated: doc.record.updated,
+            lastConfirmedAt: doc.record.lastConfirmedAt,
+            contentHash: doc.contentHash,
+            bytes: doc.bytes,
+            conformance: doc.conformance,
+            chunks: chunks.map((chunk) => ({
+                ordinal: chunk.ordinal,
+                heading: chunk.heading,
+                text: chunk.text,
+                normalizedText: normalizeForSearch(chunk.text),
+            })),
+        });
+        for (const link of doc.links) {
+            const { resolved, ambiguous } = resolveLink(link.targetRaw, doc.store, byId, byPath, byStem);
+            if (ambiguous)
+                ambiguousLinks.push({ source: doc.storePath, target: link.targetRaw });
+            else if (!resolved)
+                danglingLinks.push({ source: doc.storePath, target: link.targetRaw });
+            linkInputs.push({
+                sourcePath: doc.storePath,
+                targetRaw: link.targetRaw,
+                targetPath: resolved,
+            });
+        }
+    }
+    const corpusHash = createHash("sha256")
+        .update(walked.map((doc) => `${doc.storePath}:${doc.contentHash}`).join("\n"))
+        .digest("hex");
+    const outDir = resolveWithin(root, INDEX_DIR);
+    const outFile = path.join(outDir, "brain.sqlite");
+    await mkdir(outDir, { recursive: true });
+    await writeFile(path.join(outDir, ".gitignore"), "*\n");
+    await options.indexPort.buildIndex(docInputs, linkInputs, outFile, {
+        corpusHash,
+        embedder,
+    });
+    let legacyIndexPresent = false;
+    try {
+        await stat(resolveWithin(root, KNOWLEDGE_DIR, "index.sqlite3"));
+        legacyIndexPresent = true;
+    }
+    catch {
+        legacyIndexPresent = false;
+    }
+    return {
+        docs: docInputs.length,
+        chunks: chunkCount,
+        links: linkInputs.length,
+        danglingLinks,
+        ambiguousLinks,
+        conformance: conformanceTally,
+        scopes: scopeTally,
+        invalidScopes,
+        legacyIndexPresent,
+        vectors: Boolean(embedder),
+        outFile,
+    };
+}
+export function brainIndexPath(root = oceanRoot()) {
+    return path.join(root, INDEX_DIR, "brain.sqlite");
+}
+//# sourceMappingURL=brain-reindex.js.map