noteconnection 1.7.0 → 1.9.0

package/dist/src/routes/knowledgePayload.js

@@ -0,0 +1,185 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KnowledgePayloadError = void 0;
+exports.parseKnowledgeIngestBody = parseKnowledgeIngestBody;
+const MAX_DOCUMENTS_PER_REQUEST = 5000;
+const MAX_ALIASES_PER_DOCUMENT = 64;
+const MAX_ALIAS_LENGTH = 2048;
+const MAX_CONTENT_BYTES = 64 * 1024 * 1024;
+class KnowledgePayloadError extends Error {
+    constructor(message) {
+        super(message);
+        this.statusCode = 400;
+        this.code = 'invalid_knowledge_payload';
+        this.name = 'KnowledgePayloadError';
+    }
+}
+exports.KnowledgePayloadError = KnowledgePayloadError;
+function isRecord(value) {
+    return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+function firstString(record, keys) {
+    for (const key of keys) {
+        const value = record[key];
+        if (typeof value === 'string' && value.trim().length > 0) {
+            return value.trim();
+        }
+    }
+    return undefined;
+}
+function firstValue(record, keys) {
+    for (const key of keys) {
+        if (Object.prototype.hasOwnProperty.call(record, key)) {
+            return record[key];
+        }
+    }
+    return undefined;
+}
+function normalizeStringArray(value, field) {
+    if (value === undefined || value === null) {
+        return [];
+    }
+    if (!Array.isArray(value)) {
+        throw new KnowledgePayloadError(`${field} must be an array of strings.`);
+    }
+    if (value.length > MAX_ALIASES_PER_DOCUMENT) {
+        throw new KnowledgePayloadError(`${field} exceeds ${MAX_ALIASES_PER_DOCUMENT} entries.`);
+    }
+    const values = Array.from(new Set(value.map((item, index) => {
+        if (typeof item !== 'string' || item.trim().length === 0) {
+            throw new KnowledgePayloadError(`${field}[${index}] must be a non-empty string.`);
+        }
+        const normalized = item.trim();
+        if (normalized.length > MAX_ALIAS_LENGTH) {
+            throw new KnowledgePayloadError(`${field}[${index}] exceeds ${MAX_ALIAS_LENGTH} characters.`);
+        }
+        return normalized;
+    })));
+    return values;
+}
+function normalizeDocument(raw) {
+    if (!isRecord(raw)) {
+        throw new KnowledgePayloadError('Each ingest document must be an object.');
+    }
+    const contentRaw = firstValue(raw, ['content', 'text', 'body', 'markdown']);
+    const content = typeof contentRaw === 'string' ? contentRaw : String(contentRaw ?? '');
+    if (Buffer.byteLength(content, 'utf8') > MAX_CONTENT_BYTES) {
+        throw new KnowledgePayloadError(`Document content exceeds ${MAX_CONTENT_BYTES} bytes.`);
+    }
+    if (!content.trim() && !firstString(raw, ['documentId', 'docId', 'id', 'sourcePath', 'source_path', 'path'])) {
+        throw new KnowledgePayloadError('Document requires content or an identity.');
+    }
+    const identityAliases = normalizeStringArray(firstValue(raw, ['identityAliases', 'identity_aliases', 'aliases']), 'identityAliases');
+    return {
+        documentId: firstString(raw, ['documentId', 'docId', 'id']),
+        sourcePath: firstString(raw, ['sourcePath', 'source_path', 'path', 'filePath', 'filepath', 'file']) || '',
+        sourceUri: firstString(raw, ['sourceUri', 'source_uri', 'uri']),
+        revision: firstString(raw, ['revision', 'sourceRevision', 'source_revision']),
+        identityAliases: identityAliases.length > 0 ? identityAliases : undefined,
+        content,
+        language: firstString(raw, ['language', 'lang', 'locale']),
+        updatedAt: firstString(raw, ['updatedAt', 'updated_at', 'timestamp', 'now']),
+        workspaceId: firstString(raw, ['workspaceId', 'workspace_id']),
+        corpusId: firstString(raw, ['corpusId', 'corpus_id']),
+        exportProfileId: firstString(raw, ['exportProfileId', 'export_profile_id']),
+        metadata: isRecord(raw.metadata) ? { ...raw.metadata } : undefined,
+    };
+}
+function normalizeDelete(raw) {
+    if (!isRecord(raw)) {
+        throw new KnowledgePayloadError('Each deleted document must be an object.');
+    }
+    const identityAliases = normalizeStringArray(firstValue(raw, ['identityAliases', 'identity_aliases', 'aliases']), 'identityAliases');
+    const result = {
+        documentId: firstString(raw, ['documentId', 'docId', 'id']),
+        sourcePath: firstString(raw, ['sourcePath', 'source_path', 'path', 'filePath', 'filepath', 'file']),
+        sourceUri: firstString(raw, ['sourceUri', 'source_uri', 'uri']),
+        identityAliases: identityAliases.length > 0 ? identityAliases : undefined,
+    };
+    if (!result.documentId && !result.sourcePath && !result.sourceUri && identityAliases.length === 0) {
+        throw new KnowledgePayloadError('Delete requires documentId, sourcePath, sourceUri, or identityAliases.');
+    }
+    return result;
+}
+function normalizeMove(raw) {
+    if (!isRecord(raw)) {
+        throw new KnowledgePayloadError('Each move document must be an object.');
+    }
+    const toSourcePath = firstString(raw, ['toSourcePath', 'to_source_path', 'destinationPath', 'destination_path']);
+    if (!toSourcePath) {
+        throw new KnowledgePayloadError('Move requires toSourcePath.');
+    }
+    const aliases = normalizeStringArray(firstValue(raw, ['toIdentityAliases', 'to_identity_aliases', 'aliases']), 'toIdentityAliases');
+    const fromAliases = normalizeStringArray(firstValue(raw, ['fromIdentityAliases', 'from_identity_aliases', 'fromAliases']), 'fromIdentityAliases');
+    return {
+        documentId: firstString(raw, ['documentId', 'docId', 'id']),
+        fromSourcePath: firstString(raw, ['fromSourcePath', 'from_source_path', 'sourcePath', 'source_path']),
+        fromSourceUri: firstString(raw, ['fromSourceUri', 'from_source_uri', 'sourceUri', 'source_uri']),
+        fromIdentityAliases: fromAliases.length > 0 ? fromAliases : undefined,
+        toSourcePath,
+        toSourceUri: firstString(raw, ['toSourceUri', 'to_source_uri', 'destinationUri', 'destination_uri']),
+        toIdentityAliases: aliases.length > 0 ? aliases : undefined,
+        revision: firstString(raw, ['revision', 'sourceRevision', 'source_revision']),
+        updatedAt: firstString(raw, ['updatedAt', 'updated_at', 'timestamp', 'now']),
+    };
+}
+function normalizeOperation(raw) {
+    if (!isRecord(raw)) {
+        throw new KnowledgePayloadError('Each ingest operation must be an object.');
+    }
+    const operation = (firstString(raw, ['op', 'operation', 'action', 'type']) || 'upsert').toLowerCase();
+    const nested = firstValue(raw, ['document', 'payload', 'item']) ?? raw;
+    if (operation === 'delete' || operation === 'remove' || operation === 'del') {
+        return { op: 'delete', document: normalizeDelete(nested) };
+    }
+    if (operation === 'move' || operation === 'rename') {
+        return { op: 'move', document: normalizeMove(nested) };
+    }
+    if (operation === 'upsert' || operation === 'insert' || operation === 'update' || operation === 'put') {
+        return { op: 'upsert', document: normalizeDocument(nested) };
+    }
+    throw new KnowledgePayloadError(`Unsupported ingest operation: ${operation}.`);
+}
+function asArray(value) {
+    if (value === undefined || value === null)
+        return [];
+    return Array.isArray(value) ? value : [value];
+}
+function parseKnowledgeIngestBody(body) {
+    let payload;
+    try {
+        payload = body.trim() ? JSON.parse(body) : {};
+    }
+    catch (_error) {
+        throw new KnowledgePayloadError('Request body must be valid JSON.');
+    }
+    if (!isRecord(payload) && !Array.isArray(payload)) {
+        throw new KnowledgePayloadError('Request body must be an object or document array.');
+    }
+    const record = isRecord(payload) ? payload : {};
+    const rawDocuments = firstValue(record, ['documents', 'docs', 'items']);
+    const rawDeletes = firstValue(record, ['deletedDocuments', 'deleted', 'deletes']);
+    const rawOperations = firstValue(record, ['operations', 'ops']);
+    const documents = asArray(rawDocuments ?? (Array.isArray(payload) ? payload : undefined)).map(normalizeDocument);
+    const deletedDocuments = asArray(rawDeletes).map(normalizeDelete);
+    const operations = asArray(rawOperations).map(normalizeOperation);
+    const totalItems = documents.length + deletedDocuments.length + operations.length;
+    if (totalItems > MAX_DOCUMENTS_PER_REQUEST) {
+        throw new KnowledgePayloadError(`Ingest request exceeds ${MAX_DOCUMENTS_PER_REQUEST} documents/operations.`);
+    }
+    const incremental = typeof record.incremental === 'boolean' ? record.incremental : undefined;
+    const recomputeRelations = typeof record.recomputeRelations === 'boolean' ? record.recomputeRelations : undefined;
+    const relationRecomputeMode = firstString(record, ['relationRecomputeMode', 'relation_recompute_mode', 'recomputeMode']);
+    if (relationRecomputeMode && !['auto', 'none', 'incremental', 'full'].includes(relationRecomputeMode.toLowerCase())) {
+        throw new KnowledgePayloadError('relationRecomputeMode must be one of auto, none, incremental, full.');
+    }
+    return {
+        documents: documents.length > 0 ? documents : undefined,
+        deletedDocuments: deletedDocuments.length > 0 ? deletedDocuments : undefined,
+        operations: operations.length > 0 ? operations : undefined,
+        incremental,
+        recomputeRelations,
+        relationRecomputeMode: relationRecomputeMode?.toLowerCase(),
+        ingestedAt: firstString(record, ['ingestedAt', 'timestamp', 'now']),
+    };
+}

package/dist/src/routes/knowledgePayload.test.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const knowledgePayload_1 = require("./knowledgePayload");
+describe('knowledge HTTP payload contract', () => {
+    test('normalizes legacy aliases while preserving portable identity fields', () => {
+        const request = (0, knowledgePayload_1.parseKnowledgeIngestBody)(JSON.stringify({
+            docs: [{
+                    id: 'legacy-doc',
+                    path: 'Knowledge_Base/one.md',
+                    text: '# One',
+                    uri: 'note://workspace/v1/one.md',
+                    source_revision: 'sha256:one',
+                    aliases: ['one', 'notes/one.md'],
+                }],
+        }));
+        expect(request.documents?.[0]).toEqual(expect.objectContaining({
+            documentId: 'legacy-doc',
+            sourcePath: 'Knowledge_Base/one.md',
+            sourceUri: 'note://workspace/v1/one.md',
+            revision: 'sha256:one',
+            identityAliases: ['one', 'notes/one.md'],
+            content: '# One',
+        }));
+    });
+    test('accepts explicit move operations for replay', () => {
+        const request = (0, knowledgePayload_1.parseKnowledgeIngestBody)(JSON.stringify({
+            operations: [{
+                    op: 'rename',
+                    document: {
+                        documentId: 'doc-1',
+                        fromSourcePath: 'old.md',
+                        toSourcePath: 'new.md',
+                        toSourceUri: 'note://workspace/v1/new.md',
+                    },
+                }],
+        }));
+        expect(request.operations).toEqual([expect.objectContaining({
+                op: 'move',
+                document: expect.objectContaining({
+                    documentId: 'doc-1',
+                    toSourcePath: 'new.md',
+                }),
+            })]);
+    });
+    test('rejects malformed identity arrays and oversized documents at the edge', () => {
+        expect(() => (0, knowledgePayload_1.parseKnowledgeIngestBody)(JSON.stringify({
+            documents: [{ content: '# invalid', aliases: ['ok', 3] }],
+        }))).toThrow(knowledgePayload_1.KnowledgePayloadError);
+        expect(() => (0, knowledgePayload_1.parseKnowledgeIngestBody)(JSON.stringify({
+            documents: [{ content: 'x'.repeat(64 * 1024 * 1024 + 1) }],
+        }))).toThrow(/exceeds 67108864 bytes/);
+    });
+});

package/dist/src/routes/markdown.js

@@ -0,0 +1,295 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerMarkdownRoutes = registerMarkdownRoutes;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const CrashLogger_1 = require("../backend/utils/CrashLogger");
+const MarkdownGateway_1 = require("../markdown/MarkdownGateway");
+const PROTOCOL_VERSION = '1.0.0';
+const INDEXES = new Map();
+function buildIndexId(filePath) {
+    return `md-idx-${Buffer.from(filePath).toString('base64').slice(0, 16)}-${Date.now()}`;
+}
+function indexMarkdownFile(filePath) {
+    const resolved = path.resolve(filePath);
+    const content = fs.readFileSync(resolved, 'utf8');
+    const lines = content.split('\n');
+    const blocks = [];
+    let currentBlock = [];
+    let currentHeading;
+    let currentHeadingLevel = 0;
+    let blockStartLine = 0;
+    let blockId = 0;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const headingMatch = line.match(/^(#{1,6})\s+(.+)/);
+        if (headingMatch) {
+            // Flush current block
+            if (currentBlock.length > 0 || currentHeading) {
+                blocks.push({
+                    id: blockId++,
+                    text: currentBlock.join('\n').trim(),
+                    heading: currentHeading,
+                    headingLevel: currentHeadingLevel,
+                    startLine: blockStartLine,
+                    endLine: i - 1,
+                });
+            }
+            currentHeading = headingMatch[2].trim();
+            currentHeadingLevel = headingMatch[1].length;
+            currentBlock = [];
+            blockStartLine = i;
+        }
+        else {
+            currentBlock.push(line);
+        }
+    }
+    // Flush final block
+    if (currentBlock.length > 0 || currentHeading) {
+        blocks.push({
+            id: blockId,
+            text: currentBlock.join('\n').trim(),
+            heading: currentHeading,
+            headingLevel: currentHeadingLevel,
+            startLine: blockStartLine,
+            endLine: lines.length - 1,
+        });
+    }
+    // Ensure at least one block for empty files
+    if (blocks.length === 0) {
+        blocks.push({
+            id: 0,
+            text: content.trim() || '(empty)',
+            heading: undefined,
+            headingLevel: 0,
+            startLine: 0,
+            endLine: lines.length - 1,
+        });
+    }
+    const indexId = buildIndexId(filePath);
+    const index = {
+        indexId,
+        filePath,
+        blocks,
+        markdownProtocolVersion: PROTOCOL_VERSION,
+        indexedAt: new Date().toISOString(),
+    };
+    INDEXES.set(indexId, index);
+    return index;
+}
+function resolveNodeFromIndex(index, nodeId) {
+    const block = index.blocks.find(b => b.heading?.toLowerCase() === nodeId.toLowerCase() ||
+        b.text.toLowerCase().includes(nodeId.toLowerCase()));
+    return block ? { targetBlockId: block.id, filePath: index.filePath } : null;
+}
+function resolveWikiFromIndex(index, wikiTarget) {
+    const clean = wikiTarget.replace(/^\[\[|\]\]$/g, '').trim();
+    const exact = index.blocks.find(b => b.heading?.toLowerCase() === clean.toLowerCase());
+    if (exact)
+        return { matchType: 'exact', filePath: index.filePath, targetBlockId: exact.id };
+    const partial = index.blocks.find(b => b.heading?.toLowerCase().includes(clean.toLowerCase()) ||
+        b.text.toLowerCase().includes(clean.toLowerCase()));
+    if (partial)
+        return { matchType: 'alias', filePath: index.filePath, targetBlockId: partial.id };
+    return { matchType: 'fallback', filePath: index.filePath };
+}
+function registerMarkdownRoutes(_ctx) {
+    const ctx = _ctx;
+    const api = (p) => `/api/markdown${p}`;
+    const json = (res, code, data) => {
+        res.writeHead(code, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify(data));
+    };
+    const ok = (res, data) => json(res, 200, { success: true, ...data });
+    const fail = (res, error, label) => {
+        console.error(error);
+        CrashLogger_1.CrashLogger.log(error, label);
+        json(res, 500, { success: false, error: String(error) });
+    };
+    const readBody = (req) => new Promise((resolve, reject) => {
+        const chunks = [];
+        req.on('data', (c) => chunks.push(c));
+        req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+        req.on('error', reject);
+    });
+    const resolveRuntimeConfig = async () => {
+        if (ctx.loadFrontendSettings) {
+            const frontendSettings = await ctx.loadFrontendSettings();
+            return (0, MarkdownGateway_1.normalizeMarkdownRuntimeConfig)(frontendSettings?.reading);
+        }
+        return (0, MarkdownGateway_1.normalizeMarkdownRuntimeConfig)({});
+    };
+    return [
+        {
+            method: 'POST',
+            path: api('/index'),
+            handler: async (req, res) => {
+                try {
+                    const body = await readBody(req);
+                    const payload = JSON.parse(body);
+                    const filePath = payload.filePath || payload.path || '';
+                    if (!filePath) {
+                        json(res, 400, {
+                            success: false,
+                            error: 'Missing filePath for /api/markdown/index',
+                            markdownProtocolVersion: PROTOCOL_VERSION,
+                        });
+                        return;
+                    }
+                    if (ctx.markdownGateway) {
+                        const runtimeConfig = await resolveRuntimeConfig();
+                        const result = await ctx.markdownGateway.buildIndex({
+                            filePath,
+                            forceRebuild: payload.forceRebuild === true,
+                        }, runtimeConfig);
+                        ok(res, result);
+                        return;
+                    }
+                    if (!filePath || !fs.existsSync(filePath)) {
+                        return fail(res, new Error('File not found: ' + filePath), 'API:POST /api/markdown/index');
+                    }
+                    const index = indexMarkdownFile(filePath);
+                    ok(res, {
+                        indexId: index.indexId,
+                        filePath: index.filePath,
+                        blocksSummary: { totalBlocks: index.blocks.length },
+                        markdownProtocolVersion: index.markdownProtocolVersion,
+                    });
+                }
+                catch (e) {
+                    fail(res, e, 'API:POST /api/markdown/index');
+                }
+            },
+        },
+        {
+            method: 'POST',
+            path: api('/chunk'),
+            handler: async (req, res) => {
+                try {
+                    const body = await readBody(req);
+                    const payload = JSON.parse(body);
+                    if (ctx.markdownGateway) {
+                        const result = await ctx.markdownGateway.getChunk({
+                            indexId: String(payload.indexId || ''),
+                            startBlock: Number(payload.startBlock) || 0,
+                            blockCount: Number(payload.blockCount) || 1,
+                        });
+                        ok(res, result);
+                        return;
+                    }
+                    const index = INDEXES.get(payload.indexId || '');
+                    if (!index)
+                        return fail(res, new Error('Index not found'), 'API:POST /api/markdown/chunk');
+                    const start = payload.startBlock || 0;
+                    const count = payload.blockCount || index.blocks.length;
+                    const blocks = index.blocks.slice(start, start + count).map(b => ({
+                        blockId: b.id,
+                        text: b.text,
+                        heading: b.heading,
+                        headingLevel: b.headingLevel,
+                    }));
+                    ok(res, { blocks });
+                }
+                catch (e) {
+                    fail(res, e, 'API:POST /api/markdown/chunk');
+                }
+            },
+        },
+        {
+            method: 'POST',
+            path: api('/resolve-node'),
+            handler: async (req, res) => {
+                try {
+                    const body = await readBody(req);
+                    const payload = JSON.parse(body);
+                    if (ctx.markdownGateway) {
+                        const runtimeConfig = await resolveRuntimeConfig();
+                        const result = await ctx.markdownGateway.resolveNode({
+                            nodeId: String(payload.nodeId || ''),
+                            currentFilePath: String(payload.currentFilePath || '').trim() || undefined,
+                        }, runtimeConfig);
+                        ok(res, result);
+                        return;
+                    }
+                    const index = INDEXES.get(payload.indexId || '');
+                    // Fallback: search all indexes
+                    const searchIndex = index || [...INDEXES.values()].find(i => i.filePath === payload.currentFilePath);
+                    if (!searchIndex && INDEXES.size > 0) {
+                        return ok(res, { filePath: payload.currentFilePath || '', targetBlockId: 0 });
+                    }
+                    if (!searchIndex)
+                        return fail(res, new Error('No index available'), 'API:POST /api/markdown/resolve-node');
+                    const result = resolveNodeFromIndex(searchIndex, payload.nodeId || '');
+                    ok(res, result || { filePath: searchIndex.filePath, targetBlockId: 0 });
+                }
+                catch (e) {
+                    fail(res, e, 'API:POST /api/markdown/resolve-node');
+                }
+            },
+        },
+        {
+            method: 'POST',
+            path: api('/resolve-wiki'),
+            handler: async (req, res) => {
+                try {
+                    const body = await readBody(req);
+                    const payload = JSON.parse(body);
+                    if (ctx.markdownGateway) {
+                        const runtimeConfig = await resolveRuntimeConfig();
+                        const result = await ctx.markdownGateway.resolveWiki({
+                            wikiTarget: String(payload.wikiTarget || ''),
+                            currentFilePath: String(payload.currentFilePath || ''),
+                        }, runtimeConfig);
+                        ok(res, result);
+                        return;
+                    }
+                    const index = INDEXES.get(payload.indexId || '');
+                    const searchIndex = index || [...INDEXES.values()].find(i => i.filePath === payload.currentFilePath);
+                    if (!searchIndex && INDEXES.size > 0) {
+                        return ok(res, { filePath: payload.currentFilePath || '', matchType: 'fallback' });
+                    }
+                    if (!searchIndex)
+                        return fail(res, new Error('No index available'), 'API:POST /api/markdown/resolve-wiki');
+                    const result = resolveWikiFromIndex(searchIndex, payload.wikiTarget || '');
+                    ok(res, result || { filePath: searchIndex.filePath, matchType: 'fallback' });
+                }
+                catch (e) {
+                    fail(res, e, 'API:POST /api/markdown/resolve-wiki');
+                }
+            },
+        },
+    ];
+}