@undefineds.co/xpod 0.3.58 → 0.3.62

package/dist/storage/rdf/RdfTextProjection.js

@@ -0,0 +1,258 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rdfTextIndexPolicyRole = rdfTextIndexPolicyRole;
+exports.createRdfEntityTextChunks = createRdfEntityTextChunks;
+exports.createRdfEntityTextChunksFromText = createRdfEntityTextChunksFromText;
+exports.parseRdfTextProjectionQuads = parseRdfTextProjectionQuads;
+const node_stream_1 = require("node:stream");
+const node_crypto_1 = require("node:crypto");
+const arrayify_stream_1 = __importDefault(require("arrayify-stream"));
+const jsonld_1 = __importDefault(require("jsonld"));
+const n3_1 = require("n3");
+const rdf_parse_1 = require("rdf-parse");
+const RdfContentTypes_1 = require("./RdfContentTypes");
+const XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string';
+const RDF_LANG_STRING = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#langString';
+const SEARCHABLE_TEXT_LOCAL_NAMES = new Set([
+    'abstract',
+    'acceptance',
+    'altlabel',
+    'articlebody',
+    'body',
+    'comment',
+    'content',
+    'description',
+    'label',
+    'name',
+    'note',
+    'preflabel',
+    'summary',
+    'text',
+    'title',
+]);
+const HEADING_TEXT_LOCAL_NAMES = new Set([
+    'altlabel',
+    'label',
+    'name',
+    'preflabel',
+    'title',
+]);
+const STRUCTURED_LOCAL_NAMES = new Set([
+    'created',
+    'createdat',
+    'datecreated',
+    'datemodified',
+    'modified',
+    'modifiedat',
+    'order',
+    'position',
+    'priority',
+    'rank',
+    'score',
+    'status',
+]);
+const RELATION_LOCAL_NAMES = new Set([
+    'author',
+    'creator',
+    'editor',
+    'follows',
+    'haspart',
+    'ispartof',
+    'knows',
+    'member',
+    'mentions',
+    'owner',
+    'parent',
+    'relatedto',
+    'replyto',
+]);
+const SENSITIVE_LOCAL_NAME_PARTS = [
+    'apikey',
+    'api_key',
+    'accesskey',
+    'access_key',
+    'authorization',
+    'credential',
+    'password',
+    'privatekey',
+    'private_key',
+    'proxy',
+    'secret',
+    'token',
+];
+const SYSTEM_PREDICATE_PREFIXES = [
+    'http://www.w3.org/ns/auth/acl#',
+    'http://www.w3.org/ns/solid/acp#',
+    'http://www.w3.org/ns/solid/terms#',
+    'https://vocab.xpod.dev/credential#',
+    'https://vocab.xpod.dev/ai#',
+];
+function rdfTextIndexPolicyRole(predicate) {
+    const normalized = predicate.trim().toLowerCase();
+    if (!normalized) {
+        return 'system';
+    }
+    if (SYSTEM_PREDICATE_PREFIXES.some((prefix) => normalized.startsWith(prefix))) {
+        return 'system';
+    }
+    const localName = rdfPredicateLocalName(normalized).replace(/[-_\s]/g, '');
+    if (SENSITIVE_LOCAL_NAME_PARTS.some((part) => localName.includes(part.replace(/[-_\s]/g, '')))) {
+        return 'sensitiveText';
+    }
+    if (SEARCHABLE_TEXT_LOCAL_NAMES.has(localName)) {
+        return 'searchableText';
+    }
+    if (STRUCTURED_LOCAL_NAMES.has(localName)) {
+        return 'structured';
+    }
+    if (RELATION_LOCAL_NAMES.has(localName)) {
+        return 'relation';
+    }
+    return 'displayOnlyText';
+}
+function createRdfEntityTextChunks(source, quads, options = {}) {
+    const fieldsBySubject = new Map();
+    for (const quad of quads) {
+        if (quad.object.termType !== 'Literal' || !isSearchableLiteral(quad.object.datatype.value, quad.object.language)) {
+            continue;
+        }
+        if (quad.predicate.termType !== 'NamedNode') {
+            continue;
+        }
+        const role = rdfTextIndexPolicyRole(quad.predicate.value);
+        if (role !== 'searchableText') {
+            continue;
+        }
+        const subject = quad.subject.value;
+        const fields = fieldsBySubject.get(subject) ?? [];
+        fields.push({
+            predicate: quad.predicate.value,
+            label: rdfPredicateDisplayLabel(quad.predicate.value),
+            value: quad.object.value,
+            datatype: quad.object.datatype.value,
+            language: quad.object.language,
+            policyRole: role,
+        });
+        fieldsBySubject.set(subject, fields);
+    }
+    const chunks = [];
+    let ordinal = 0;
+    for (const [subject, fields] of [...fieldsBySubject.entries()].sort(([left], [right]) => left.localeCompare(right))) {
+        const orderedFields = fields.sort((left, right) => left.predicate.localeCompare(right.predicate) || left.value.localeCompare(right.value));
+        const cardFields = orderedFields.filter((field) => !isLongRdfTextField(field.value, options.maxFieldBytes));
+        const fieldChunkFields = orderedFields.filter((field) => isLongRdfTextField(field.value, options.maxFieldBytes));
+        if (cardFields.length > 0) {
+            const content = cardFields
+                .map((field) => `${field.label}: ${field.value}`)
+                .join('\n');
+            chunks.push({
+                chunkKey: rdfEntityChunkKey(source.source, subject),
+                retrievalKind: 'entity-card',
+                ordinal: ordinal++,
+                level: 0,
+                heading: rdfEntityHeading(cardFields),
+                path: [],
+                content,
+                startOffset: 0,
+                endOffset: content.length,
+                entities: cardFields.map((field) => rdfTextFieldEntity(subject, field)),
+            });
+        }
+        for (const field of fieldChunkFields) {
+            const content = `${field.label}: ${field.value}`;
+            chunks.push({
+                chunkKey: rdfFieldChunkKey(source.source, subject, field.predicate, field.value),
+                retrievalKind: 'field-chunk',
+                ordinal: ordinal++,
+                level: 0,
+                heading: rdfEntityHeading(cardFields),
+                path: [],
+                content,
+                startOffset: 0,
+                endOffset: content.length,
+                entities: [rdfTextFieldEntity(subject, field)],
+            });
+        }
+    }
+    return chunks;
+}
+async function createRdfEntityTextChunksFromText(source, text, options = {}) {
+    return createRdfEntityTextChunks(source, await parseRdfTextProjectionQuads(source, text), options);
+}
+function isLongRdfTextField(value, maxFieldBytes) {
+    if (maxFieldBytes === undefined || !Number.isFinite(maxFieldBytes)) {
+        return false;
+    }
+    return Buffer.byteLength(value) > Math.max(0, Math.trunc(maxFieldBytes));
+}
+function rdfEntityHeading(fields) {
+    return fields.find((field) => HEADING_TEXT_LOCAL_NAMES.has(rdfPredicateLocalName(field.predicate).toLowerCase().replace(/[-_\s]/g, '')))?.value;
+}
+function rdfTextFieldEntity(subject, field) {
+    return {
+        entity: subject,
+        predicate: field.predicate,
+        value: field.value,
+        datatype: field.datatype,
+        language: field.language || undefined,
+        policyRole: field.policyRole,
+    };
+}
+async function parseRdfTextProjectionQuads(source, text) {
+    const contentType = (0, RdfContentTypes_1.normalizeContentType)(source.contentType) ?? (0, RdfContentTypes_1.rdfContentTypeForPath)(source.localPath ?? source.source) ?? 'text/turtle';
+    if (contentType === 'application/ld+json') {
+        const nquads = await jsonld_1.default.toRDF(JSON.parse(text), {
+            base: source.source,
+            format: 'application/n-quads',
+        });
+        return new n3_1.Parser({ format: 'application/n-quads', baseIRI: source.source }).parse(nquads);
+    }
+    if (contentType === 'application/rdf+xml') {
+        return (0, arrayify_stream_1.default)(rdf_parse_1.rdfParser.parse(node_stream_1.Readable.from([text]), {
+            contentType,
+            baseIRI: source.source,
+        }));
+    }
+    return new n3_1.Parser({ format: contentType, baseIRI: source.source }).parse(text);
+}
+function isSearchableLiteral(datatype, language) {
+    return datatype === XSD_STRING || datatype === RDF_LANG_STRING || Boolean(language);
+}
+function rdfEntityChunkKey(source, subject) {
+    return (0, node_crypto_1.createHash)('sha256')
+        .update(source)
+        .update('\0rdf-entity\0')
+        .update(subject)
+        .digest('hex')
+        .slice(0, 24);
+}
+function rdfFieldChunkKey(source, subject, predicate, value) {
+    return (0, node_crypto_1.createHash)('sha256')
+        .update(source)
+        .update('\0rdf-field\0')
+        .update(subject)
+        .update('\0')
+        .update(predicate)
+        .update('\0')
+        .update(value)
+        .digest('hex')
+        .slice(0, 24);
+}
+function rdfPredicateDisplayLabel(predicate) {
+    const local = rdfPredicateLocalName(predicate);
+    return local
+        .replace(/[_-]+/g, ' ')
+        .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+        .trim() || 'text';
+}
+function rdfPredicateLocalName(predicate) {
+    const hash = predicate.lastIndexOf('#');
+    const slash = predicate.lastIndexOf('/');
+    const colon = predicate.lastIndexOf(':');
+    const index = Math.max(hash, slash, colon);
+    return index >= 0 ? predicate.slice(index + 1) : predicate;
+}
+//# sourceMappingURL=RdfTextProjection.js.map

package/dist/storage/rdf/RdfTextProjection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RdfTextProjection.js","sourceRoot":"","sources":["../../../src/storage/rdf/RdfTextProjection.ts"],"names":[],"mappings":";;;;;AAqGA,wDAuBC;AAED,8DAuEC;AAED,8EAMC;AAuCD,kEAgBC;AApQD,6CAAuC;AACvC,6CAAyC;AACzC,sEAA6C;AAC7C,oDAA4B;AAC5B,2BAA4B;AAC5B,yCAAsC;AAGtC,uDAAgF;AAEhF,MAAM,UAAU,GAAG,yCAAyC,CAAC;AAC7D,MAAM,eAAe,GAAG,uDAAuD,CAAC;AAUhF,MAAM,2BAA2B,GAAG,IAAI,GAAG,CAAC;IAC1C,UAAU;IACV,YAAY;IACZ,UAAU;IACV,aAAa;IACb,MAAM;IACN,SAAS;IACT,SAAS;IACT,aAAa;IACb,OAAO;IACP,MAAM;IACN,MAAM;IACN,WAAW;IACX,SAAS;IACT,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,UAAU;IACV,OAAO;IACP,MAAM;IACN,WAAW;IACX,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,SAAS;IACT,WAAW;IACX,aAAa;IACb,cAAc;IACd,UAAU;IACV,YAAY;IACZ,OAAO;IACP,UAAU;IACV,UAAU;IACV,MAAM;IACN,OAAO;IACP,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,QAAQ;IACR,SAAS;IACT,QAAQ;IACR,SAAS;IACT,SAAS;IACT,UAAU;IACV,OAAO;IACP,QAAQ;IACR,UAAU;IACV,OAAO;IACP,QAAQ;IACR,WAAW;IACX,SAAS;CACV,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAAG;IACjC,QAAQ;IACR,SAAS;IACT,WAAW;IACX,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,aAAa;IACb,OAAO;IACP,QAAQ;IACR,OAAO;CACR,CAAC;AAEF,MAAM,yBAAyB,GAAG;IAChC,gCAAgC;IAChC,iCAAiC;IACjC,mCAAmC;IACnC,oCAAoC;IACpC,4BAA4B;CAC7B,CAAC;AAEF,SAAgB,sBAAsB,CAAC,SAAiB;IACtD,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAClD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,yBAAyB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QAC9E,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC3E,IAAI,0BAA0B,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,OAAO,eAAe,CAAC;IACzB,CAAC;IACD,IAAI,2BAA2B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/C,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,IAAI,sBAAsB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1C,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,SAAgB,yBAAyB,CACvC,MAA0B,EAC1B,KAAa,EACb,UAA0C,EAAE;IAE5C,MAAM,eAAe,GAAG,IAAI,GAAG,EAAoC,CAAC;IACpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjH,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;YAC5C,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1D,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;QACnC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC;YACV,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK;YAC/B,KAAK,EAAE,wBAAwB,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;YACrD,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK;YACpC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YAC9B,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QACH,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACpH,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAChD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1F,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;QAC5G,MAAM,gBAAgB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;QACjH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,UAAU;iBACvB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;iBAChD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;gBACnD,aAAa,EAAE,aAAa;gBAC5B,OAAO,EAAE,OAAO,EAAE;gBAClB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,CAAC,UAAU,CAAC;gBACrC,IAAI,EAAE,EAAE;gBACR,OAAO;gBACP,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,OAAO,CAAC,MAAM;gBACzB,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;aACxE,CAAC,CAAC;QACL,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC;gBAChF,aAAa,EAAE,aAAa;gBAC5B,OAAO,EAAE,OAAO,EAAE;gBAClB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,CAAC,UAAU,CAAC;gBACrC,IAAI,EAAE,EAAE;gBACR,OAAO;gBACP,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,OAAO,CAAC,MAAM;gBACzB,QAAQ,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,iCAAiC,CACrD,MAA0B,EAC1B,IAAY,EACZ,UAA0C,EAAE;IAE5C,OAAO,yBAAyB,CAAC,MAAM,EAAE,MAAM,2BAA2B,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;AACrG,CAAC;AAeD,SAAS,kBAAkB,CAAC,KAAa,EAAE,aAAiC;IAC1E,IAAI,aAAa,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACnE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAgC;IACxD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAC3B,wBAAwB,CAAC,GAAG,CAAC,qBAAqB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAC1G,EAAE,KAAK,CAAC;AACX,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAe,EAAE,KAA6B;IACxE,OAAO;QACL,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,SAAS;QACrC,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,2BAA2B,CAAC,MAA0B,EAAE,IAAY;IACxF,MAAM,WAAW,GAAG,IAAA,sCAAoB,EAAC,MAAM,CAAC,WAAW,CAAC,IAAI,IAAA,uCAAqB,EAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,aAAa,CAAC;IAC1I,IAAI,WAAW,KAAK,qBAAqB,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,MAAM,gBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;YAClD,IAAI,EAAE,MAAM,CAAC,MAAM;YACnB,MAAM,EAAE,qBAAqB;SAC9B,CAAW,CAAC;QACb,OAAO,IAAI,WAAM,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,WAAW,KAAK,qBAAqB,EAAE,CAAC;QAC1C,OAAO,IAAA,yBAAc,EAAO,qBAAS,CAAC,KAAK,CAAC,sBAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE;YACjE,WAAW;YACX,OAAO,EAAE,MAAM,CAAC,MAAM;SACvB,CAAQ,CAAC,CAAC;IACb,CAAC;IACD,OAAO,IAAI,WAAM,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgB,EAAE,QAAgB;IAC7D,OAAO,QAAQ,KAAK,UAAU,IAAI,QAAQ,KAAK,eAAe,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAc,EAAE,OAAe;IACxD,OAAO,IAAA,wBAAU,EAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,MAAM,CAAC;SACd,MAAM,CAAC,gBAAgB,CAAC;SACxB,MAAM,CAAC,OAAO,CAAC;SACf,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc,EAAE,OAAe,EAAE,SAAiB,EAAE,KAAa;IACzF,OAAO,IAAA,wBAAU,EAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,MAAM,CAAC;SACd,MAAM,CAAC,eAAe,CAAC;SACvB,MAAM,CAAC,OAAO,CAAC;SACf,MAAM,CAAC,IAAI,CAAC;SACZ,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,IAAI,CAAC;SACZ,MAAM,CAAC,KAAK,CAAC;SACb,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,wBAAwB,CAAC,SAAiB;IACjD,MAAM,KAAK,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,KAAK;SACT,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC;SACtC,IAAI,EAAE,IAAI,MAAM,CAAC;AACtB,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAiB;IAC9C,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC3C,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7D,CAAC","sourcesContent":["import { Readable } from 'node:stream';\nimport { createHash } from 'node:crypto';\nimport arrayifyStream from 'arrayify-stream';\nimport jsonld from 'jsonld';\nimport { Parser } from 'n3';\nimport { rdfParser } from 'rdf-parse';\nimport type { Quad } from '@rdfjs/types';\nimport type { RdfTextChunkInput, RdfTextSourceInput } from './types';\nimport { normalizeContentType, rdfContentTypeForPath } from './RdfContentTypes';\n\nconst XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string';\nconst RDF_LANG_STRING = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#langString';\n\nexport type TextIndexPolicyRole =\n  | 'searchableText'\n  | 'displayOnlyText'\n  | 'sensitiveText'\n  | 'structured'\n  | 'relation'\n  | 'system';\n\nconst SEARCHABLE_TEXT_LOCAL_NAMES = new Set([\n  'abstract',\n  'acceptance',\n  'altlabel',\n  'articlebody',\n  'body',\n  'comment',\n  'content',\n  'description',\n  'label',\n  'name',\n  'note',\n  'preflabel',\n  'summary',\n  'text',\n  'title',\n]);\n\nconst HEADING_TEXT_LOCAL_NAMES = new Set([\n  'altlabel',\n  'label',\n  'name',\n  'preflabel',\n  'title',\n]);\n\nconst STRUCTURED_LOCAL_NAMES = new Set([\n  'created',\n  'createdat',\n  'datecreated',\n  'datemodified',\n  'modified',\n  'modifiedat',\n  'order',\n  'position',\n  'priority',\n  'rank',\n  'score',\n  'status',\n]);\n\nconst RELATION_LOCAL_NAMES = new Set([\n  'author',\n  'creator',\n  'editor',\n  'follows',\n  'haspart',\n  'ispartof',\n  'knows',\n  'member',\n  'mentions',\n  'owner',\n  'parent',\n  'relatedto',\n  'replyto',\n]);\n\nconst SENSITIVE_LOCAL_NAME_PARTS = [\n  'apikey',\n  'api_key',\n  'accesskey',\n  'access_key',\n  'authorization',\n  'credential',\n  'password',\n  'privatekey',\n  'private_key',\n  'proxy',\n  'secret',\n  'token',\n];\n\nconst SYSTEM_PREDICATE_PREFIXES = [\n  'http://www.w3.org/ns/auth/acl#',\n  'http://www.w3.org/ns/solid/acp#',\n  'http://www.w3.org/ns/solid/terms#',\n  'https://vocab.xpod.dev/credential#',\n  'https://vocab.xpod.dev/ai#',\n];\n\nexport function rdfTextIndexPolicyRole(predicate: string): TextIndexPolicyRole {\n  const normalized = predicate.trim().toLowerCase();\n  if (!normalized) {\n    return 'system';\n  }\n  if (SYSTEM_PREDICATE_PREFIXES.some((prefix) => normalized.startsWith(prefix))) {\n    return 'system';\n  }\n\n  const localName = rdfPredicateLocalName(normalized).replace(/[-_\\s]/g, '');\n  if (SENSITIVE_LOCAL_NAME_PARTS.some((part) => localName.includes(part.replace(/[-_\\s]/g, '')))) {\n    return 'sensitiveText';\n  }\n  if (SEARCHABLE_TEXT_LOCAL_NAMES.has(localName)) {\n    return 'searchableText';\n  }\n  if (STRUCTURED_LOCAL_NAMES.has(localName)) {\n    return 'structured';\n  }\n  if (RELATION_LOCAL_NAMES.has(localName)) {\n    return 'relation';\n  }\n  return 'displayOnlyText';\n}\n\nexport function createRdfEntityTextChunks(\n  source: RdfTextSourceInput,\n  quads: Quad[],\n  options: RdfEntityTextProjectionOptions = {},\n): RdfTextChunkInput[] {\n  const fieldsBySubject = new Map<string, RdfTextProjectionField[]>();\n  for (const quad of quads) {\n    if (quad.object.termType !== 'Literal' || !isSearchableLiteral(quad.object.datatype.value, quad.object.language)) {\n      continue;\n    }\n    if (quad.predicate.termType !== 'NamedNode') {\n      continue;\n    }\n    const role = rdfTextIndexPolicyRole(quad.predicate.value);\n    if (role !== 'searchableText') {\n      continue;\n    }\n    const subject = quad.subject.value;\n    const fields = fieldsBySubject.get(subject) ?? [];\n    fields.push({\n      predicate: quad.predicate.value,\n      label: rdfPredicateDisplayLabel(quad.predicate.value),\n      value: quad.object.value,\n      datatype: quad.object.datatype.value,\n      language: quad.object.language,\n      policyRole: role,\n    });\n    fieldsBySubject.set(subject, fields);\n  }\n\n  const chunks: RdfTextChunkInput[] = [];\n  let ordinal = 0;\n  for (const [subject, fields] of [...fieldsBySubject.entries()].sort(([left], [right]) => left.localeCompare(right))) {\n    const orderedFields = fields.sort((left, right) =>\n      left.predicate.localeCompare(right.predicate) || left.value.localeCompare(right.value));\n    const cardFields = orderedFields.filter((field) => !isLongRdfTextField(field.value, options.maxFieldBytes));\n    const fieldChunkFields = orderedFields.filter((field) => isLongRdfTextField(field.value, options.maxFieldBytes));\n    if (cardFields.length > 0) {\n      const content = cardFields\n        .map((field) => `${field.label}: ${field.value}`)\n        .join('\\n');\n      chunks.push({\n        chunkKey: rdfEntityChunkKey(source.source, subject),\n        retrievalKind: 'entity-card',\n        ordinal: ordinal++,\n        level: 0,\n        heading: rdfEntityHeading(cardFields),\n        path: [],\n        content,\n        startOffset: 0,\n        endOffset: content.length,\n        entities: cardFields.map((field) => rdfTextFieldEntity(subject, field)),\n      });\n    }\n    for (const field of fieldChunkFields) {\n      const content = `${field.label}: ${field.value}`;\n      chunks.push({\n        chunkKey: rdfFieldChunkKey(source.source, subject, field.predicate, field.value),\n        retrievalKind: 'field-chunk',\n        ordinal: ordinal++,\n        level: 0,\n        heading: rdfEntityHeading(cardFields),\n        path: [],\n        content,\n        startOffset: 0,\n        endOffset: content.length,\n        entities: [rdfTextFieldEntity(subject, field)],\n      });\n    }\n  }\n  return chunks;\n}\n\nexport async function createRdfEntityTextChunksFromText(\n  source: RdfTextSourceInput,\n  text: string,\n  options: RdfEntityTextProjectionOptions = {},\n): Promise<RdfTextChunkInput[]> {\n  return createRdfEntityTextChunks(source, await parseRdfTextProjectionQuads(source, text), options);\n}\n\nexport interface RdfEntityTextProjectionOptions {\n  maxFieldBytes?: number;\n}\n\ninterface RdfTextProjectionField {\n  predicate: string;\n  label: string;\n  value: string;\n  datatype: string;\n  language: string;\n  policyRole: TextIndexPolicyRole;\n}\n\nfunction isLongRdfTextField(value: string, maxFieldBytes: number | undefined): boolean {\n  if (maxFieldBytes === undefined || !Number.isFinite(maxFieldBytes)) {\n    return false;\n  }\n  return Buffer.byteLength(value) > Math.max(0, Math.trunc(maxFieldBytes));\n}\n\nfunction rdfEntityHeading(fields: RdfTextProjectionField[]): string | undefined {\n  return fields.find((field) =>\n    HEADING_TEXT_LOCAL_NAMES.has(rdfPredicateLocalName(field.predicate).toLowerCase().replace(/[-_\\s]/g, '')),\n  )?.value;\n}\n\nfunction rdfTextFieldEntity(subject: string, field: RdfTextProjectionField): NonNullable<RdfTextChunkInput['entities']>[number] {\n  return {\n    entity: subject,\n    predicate: field.predicate,\n    value: field.value,\n    datatype: field.datatype,\n    language: field.language || undefined,\n    policyRole: field.policyRole,\n  };\n}\n\nexport async function parseRdfTextProjectionQuads(source: RdfTextSourceInput, text: string): Promise<Quad[]> {\n  const contentType = normalizeContentType(source.contentType) ?? rdfContentTypeForPath(source.localPath ?? source.source) ?? 'text/turtle';\n  if (contentType === 'application/ld+json') {\n    const nquads = await jsonld.toRDF(JSON.parse(text), {\n      base: source.source,\n      format: 'application/n-quads',\n    }) as string;\n    return new Parser({ format: 'application/n-quads', baseIRI: source.source }).parse(nquads);\n  }\n  if (contentType === 'application/rdf+xml') {\n    return arrayifyStream<Quad>(rdfParser.parse(Readable.from([text]), {\n      contentType,\n      baseIRI: source.source,\n    }) as any);\n  }\n  return new Parser({ format: contentType, baseIRI: source.source }).parse(text);\n}\n\nfunction isSearchableLiteral(datatype: string, language: string): boolean {\n  return datatype === XSD_STRING || datatype === RDF_LANG_STRING || Boolean(language);\n}\n\nfunction rdfEntityChunkKey(source: string, subject: string): string {\n  return createHash('sha256')\n    .update(source)\n    .update('\\0rdf-entity\\0')\n    .update(subject)\n    .digest('hex')\n    .slice(0, 24);\n}\n\nfunction rdfFieldChunkKey(source: string, subject: string, predicate: string, value: string): string {\n  return createHash('sha256')\n    .update(source)\n    .update('\\0rdf-field\\0')\n    .update(subject)\n    .update('\\0')\n    .update(predicate)\n    .update('\\0')\n    .update(value)\n    .digest('hex')\n    .slice(0, 24);\n}\n\nfunction rdfPredicateDisplayLabel(predicate: string): string {\n  const local = rdfPredicateLocalName(predicate);\n  return local\n    .replace(/[_-]+/g, ' ')\n    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')\n    .trim() || 'text';\n}\n\nfunction rdfPredicateLocalName(predicate: string): string {\n  const hash = predicate.lastIndexOf('#');\n  const slash = predicate.lastIndexOf('/');\n  const colon = predicate.lastIndexOf(':');\n  const index = Math.max(hash, slash, colon);\n  return index >= 0 ? predicate.slice(index + 1) : predicate;\n}\n"]}

package/dist/storage/rdf/RdfVectorIndex.d.ts

@@ -1,5 +1,11 @@
-import type { RdfVectorChunkInput, RdfSearchCardinalityEstimate, RdfVectorIndexOptions, RdfVectorIndexStats, RdfVectorModelDistribution, RdfVectorSearchOptions, RdfVectorSearchResult, RdfVectorSourceInput } from './types';
-export declare class RdfVectorIndex {
+import type { RdfVectorChunkInput, RdfVectorChunkRow, RdfVectorDistanceMetric, RdfSearchCardinalityEstimate, RdfVectorIndexOptions, RdfVectorIndexSyncLike, RdfVectorIndexStats, RdfVectorModelDistribution, RdfVectorSearchOrder, RdfVectorSearchOptions, RdfVectorSearchResult, RdfVectorSourceInput, RdfVectorScoreComponents, RdfVectorSummaryLifecycleEntry, RdfVectorSummaryLifecycleOptions, RdfVectorSummaryMetadata } from './types';
+export interface RdfVectorScoredChunkRow extends RdfVectorChunkRow {
+    dot_product: number;
+    vector_score: number;
+    vector_distance: number | null;
+    vector_distance_squared: number | null;
+}
+export declare class RdfVectorIndex implements RdfVectorIndexSyncLike {
     private readonly options;
     private readonly sqliteRuntime;
     private db;
@@ -10,13 +16,35 @@ export declare class RdfVectorIndex {
     indexVector(source: RdfVectorSourceInput, chunks: RdfVectorChunkInput[]): void;
     deleteSource(source: string): number;
     search(options: RdfVectorSearchOptions): RdfVectorSearchResult[];
+    summaryLifecycle(options?: RdfVectorSummaryLifecycleOptions): RdfVectorSummaryLifecycleEntry[];
     estimateSearchCardinality(options: RdfVectorSearchOptions): RdfSearchCardinalityEstimate;
     stats(): RdfVectorIndexStats;
     modelDistribution(): RdfVectorModelDistribution[];
     private initializeSchema;
+    private ensureSourceKeyColumn;
+    private ensureVectorIdentityColumns;
+    private ensureVectorChunkIdentityUniqueConstraint;
     private backfillVectorComponents;
     private upsertSource;
     private estimateDatabaseBytes;
     private toSearchResult;
     private requireDb;
 }
+export declare function vectorScoreComponents(row: Pick<RdfVectorScoredChunkRow, 'dimensions' | 'dot_product' | 'magnitude' | 'vector_distance_squared'>, metric: RdfVectorDistanceMetric, queryMagnitude: number, score: number, distance: number, backend?: 'component' | 'pg-vector'): RdfVectorScoreComponents;
+export declare function normalizeEmbedding(embedding: number[]): number[];
+export declare function parseEmbedding(value: string): number[];
+export declare function vectorMagnitude(embedding: number[]): number;
+export declare function vectorScore(distance: number, metric: RdfVectorDistanceMetric): number;
+export declare function scoredVectorDistance(row: RdfVectorScoredChunkRow, metric: RdfVectorDistanceMetric): number;
+export declare function insertVectorComponents(insertComponent: {
+    run(...params: unknown[]): unknown;
+}, chunkId: number, embedding: number[]): void;
+export declare function parsePath(value: string | null): string[];
+export declare function parseSummaryMetadata(value: string | null): RdfVectorSummaryMetadata | undefined;
+export declare function buildVectorOrderClause(metric: RdfVectorDistanceMetric, orderBy: RdfVectorSearchOrder[] | undefined): string;
+export declare function vectorScoreSql(metric: RdfVectorDistanceMetric, queryMagnitude: number): string;
+export declare function vectorDistanceSql(metric: RdfVectorDistanceMetric, queryMagnitude: number): string;
+export declare function vectorSquaredDistanceSql(queryMagnitude: number): string;
+export declare function vectorThresholdSql(metric: RdfVectorDistanceMetric, queryMagnitude: number, threshold: number): string;
+export declare function applyResultWindow(rows: number, offset: number | undefined, limit: number | undefined): number;
+export declare function sqlNumber(value: number): string;