@undefineds.co/xpod 0.3.75 → 0.4.0-rc.140

package/dist/document/ReaderMaterializationRepository.js

@@ -0,0 +1,400 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ReaderMaterializationRepository = void 0;
+exports.computeReaderMaterializationFingerprint = computeReaderMaterializationFingerprint;
+const node_crypto_1 = require("node:crypto");
+const drizzle_orm_1 = require("drizzle-orm");
+const db_1 = require("../identity/drizzle/db");
+const PG_READER_MATERIALIZATION_SCHEMA_LOCK_KEY = 1_936_528_503;
+class ReaderMaterializationRepository {
+    constructor(db) {
+        this.db = db;
+        this.ready = this.initialize();
+    }
+    async putBody(input, now = new Date()) {
+        await this.ready;
+        const fingerprint = computeReaderMaterializationFingerprint(input);
+        const createdAt = timestampText(this.db, now);
+        const insertResult = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+      INSERT INTO reader_materialization_body (
+        fingerprint,
+        source_key,
+        source_uri,
+        source_hash,
+        media_type,
+        reader_engine,
+        reader_version,
+        model_uri,
+        reader_options_hash,
+        representation_hash,
+        markdown,
+        created_at
+      ) VALUES (
+        ${fingerprint},
+        ${input.sourceKey},
+        ${input.sourceUri},
+        ${input.sourceHash},
+        ${input.mediaType},
+        ${input.readerEngine},
+        ${input.readerVersion},
+        ${input.modelUri ?? null},
+        ${input.readerOptionsHash},
+        ${input.representationHash},
+        ${input.markdown},
+        ${createdAt}
+      )
+      ON CONFLICT (fingerprint) DO NOTHING
+      RETURNING
+        fingerprint,
+        source_key,
+        source_uri,
+        source_hash,
+        media_type,
+        reader_engine,
+        reader_version,
+        model_uri,
+        reader_options_hash,
+        representation_hash,
+        markdown,
+        created_at
+    `);
+        if (insertResult.rows[0]) {
+            return mapBodyRow(insertResult.rows[0]);
+        }
+        const conflicting = await this.getBody(fingerprint);
+        if (!conflicting) {
+            throw new Error(`Failed to write reader materialization body for fingerprint ${fingerprint}`);
+        }
+        if (!isIdempotentBody(conflicting, input)) {
+            throw new Error(`Reader materialization body is immutable for fingerprint ${fingerprint}`);
+        }
+        return conflicting;
+    }
+    async getBody(fingerprint) {
+        await this.ready;
+        const result = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+      SELECT
+        fingerprint,
+        source_key,
+        source_uri,
+        source_hash,
+        media_type,
+        reader_engine,
+        reader_version,
+        model_uri,
+        reader_options_hash,
+        representation_hash,
+        markdown,
+        created_at
+      FROM reader_materialization_body
+      WHERE fingerprint = ${fingerprint}
+      LIMIT 1
+    `);
+        return result.rows[0] ? mapBodyRow(result.rows[0]) : undefined;
+    }
+    async enqueue(input, now = new Date()) {
+        await this.ready;
+        const ts = timestampText(this.db, now);
+        const result = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+      INSERT INTO reader_reconciliation (
+        source_key,
+        source_uri,
+        desired_fingerprint,
+        reason,
+        attempt_count,
+        next_attempt_at,
+        lease_owner,
+        lease_expires_at,
+        last_failure_category,
+        updated_at
+      ) VALUES (
+        ${input.sourceKey},
+        ${input.sourceUri},
+        ${input.desiredFingerprint ?? null},
+        ${input.reason},
+        0,
+        ${ts},
+        NULL,
+        NULL,
+        NULL,
+        ${ts}
+      )
+      ON CONFLICT (source_key) DO UPDATE SET
+        desired_fingerprint = excluded.desired_fingerprint,
+        reason = excluded.reason,
+        updated_at = excluded.updated_at
+      WHERE reader_reconciliation.source_uri = excluded.source_uri
+      RETURNING
+        source_key,
+        source_uri,
+        desired_fingerprint,
+        reason,
+        attempt_count,
+        next_attempt_at,
+        lease_owner,
+        lease_expires_at,
+        last_failure_category,
+        updated_at
+    `);
+        if (!result.rows[0]) {
+            throw new Error(`Reader reconciliation source URI is immutable for source key ${input.sourceKey}`);
+        }
+        return mapReconciliationRow(result.rows[0]);
+    }
+    async listRunnable(now = new Date()) {
+        await this.ready;
+        const ts = timestampText(this.db, now);
+        const result = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+      SELECT
+        source_key,
+        source_uri,
+        desired_fingerprint,
+        reason,
+        attempt_count,
+        next_attempt_at,
+        lease_owner,
+        lease_expires_at,
+        last_failure_category,
+        updated_at
+      FROM reader_reconciliation
+      WHERE next_attempt_at <= ${ts}
+        AND (lease_owner IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ${ts})
+      ORDER BY next_attempt_at ASC, source_key ASC
+    `);
+        return result.rows.map(mapReconciliationRow);
+    }
+    async claimNext(workerId, now = new Date(), leaseDurationMs = 60_000) {
+        await this.ready;
+        const ts = timestampText(this.db, now);
+        const leaseExpiresAt = timestampText(this.db, new Date(now.getTime() + leaseDurationMs));
+        const candidates = await this.listRunnable(now);
+        for (const candidate of candidates) {
+            const result = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+        UPDATE reader_reconciliation
+        SET lease_owner = ${workerId},
+            lease_expires_at = ${leaseExpiresAt},
+            updated_at = ${ts}
+        WHERE source_key = ${candidate.sourceKey}
+          AND next_attempt_at <= ${ts}
+          AND (lease_owner IS NULL OR lease_expires_at IS NULL OR lease_expires_at <= ${ts})
+        RETURNING
+          source_key,
+          source_uri,
+          desired_fingerprint,
+          reason,
+          attempt_count,
+          next_attempt_at,
+          lease_owner,
+          lease_expires_at,
+          last_failure_category,
+          updated_at
+      `);
+            if (result.rows[0]) {
+                return mapReconciliationRow(result.rows[0]);
+            }
+        }
+        return undefined;
+    }
+    async fail(sourceKey, workerId, failureCategory, nextAttemptAt, now = new Date()) {
+        await this.ready;
+        const result = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+      UPDATE reader_reconciliation
+      SET attempt_count = attempt_count + 1,
+          next_attempt_at = ${timestampText(this.db, nextAttemptAt)},
+          lease_owner = NULL,
+          lease_expires_at = NULL,
+          last_failure_category = ${failureCategory},
+          updated_at = ${timestampText(this.db, now)}
+      WHERE source_key = ${sourceKey}
+        AND lease_owner = ${workerId}
+        AND lease_expires_at > ${timestampText(this.db, now)}
+      RETURNING
+        source_key,
+        source_uri,
+        desired_fingerprint,
+        reason,
+        attempt_count,
+        next_attempt_at,
+        lease_owner,
+        lease_expires_at,
+        last_failure_category,
+        updated_at
+    `);
+        if (!result.rows[0]) {
+            throw new Error(`Reader reconciliation ownership mismatch for source key ${sourceKey}`);
+        }
+    }
+    async complete(sourceKey, workerId, now = new Date()) {
+        await this.ready;
+        const result = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+      DELETE FROM reader_reconciliation
+      WHERE source_key = ${sourceKey}
+        AND lease_owner = ${workerId}
+        AND lease_expires_at > ${timestampText(this.db, now)}
+      RETURNING
+        source_key,
+        source_uri,
+        desired_fingerprint,
+        reason,
+        attempt_count,
+        next_attempt_at,
+        lease_owner,
+        lease_expires_at,
+        last_failure_category,
+        updated_at
+    `);
+        if (!result.rows[0]) {
+            throw new Error(`Reader reconciliation ownership mismatch for source key ${sourceKey}`);
+        }
+    }
+    async releaseExpiredLeases(now = new Date()) {
+        await this.ready;
+        const ts = timestampText(this.db, now);
+        await (0, db_1.executeStatement)(this.db, (0, drizzle_orm_1.sql) `
+      UPDATE reader_reconciliation
+      SET lease_owner = NULL,
+          lease_expires_at = NULL,
+          updated_at = ${ts}
+      WHERE lease_owner IS NOT NULL
+        AND lease_expires_at IS NOT NULL
+        AND lease_expires_at <= ${ts}
+    `);
+    }
+    async moveSource(sourceKey, nextSourceUri, now = new Date()) {
+        await this.ready;
+        const ts = timestampText(this.db, now);
+        await (0, db_1.executeStatement)(this.db, (0, drizzle_orm_1.sql) `
+      UPDATE reader_materialization_body
+      SET source_uri = ${nextSourceUri}
+      WHERE source_key = ${sourceKey}
+    `);
+        await (0, db_1.executeStatement)(this.db, (0, drizzle_orm_1.sql) `
+      UPDATE reader_reconciliation
+      SET source_uri = ${nextSourceUri},
+          updated_at = ${ts}
+      WHERE source_key = ${sourceKey}
+    `);
+    }
+    async initialize() {
+        await (0, db_1.executePostgresLockedStatements)(this.db, PG_READER_MATERIALIZATION_SCHEMA_LOCK_KEY, [
+            `
+        CREATE TABLE IF NOT EXISTS reader_materialization_body (
+          fingerprint TEXT PRIMARY KEY,
+          source_key TEXT NOT NULL,
+          source_uri TEXT NOT NULL,
+          source_hash TEXT NOT NULL,
+          media_type TEXT NOT NULL,
+          reader_engine TEXT NOT NULL,
+          reader_version TEXT NOT NULL,
+          model_uri TEXT,
+          reader_options_hash TEXT NOT NULL,
+          representation_hash TEXT NOT NULL,
+          markdown TEXT NOT NULL,
+          created_at TEXT NOT NULL
+        )
+      `,
+            `
+        CREATE TABLE IF NOT EXISTS reader_reconciliation (
+          source_key TEXT PRIMARY KEY,
+          source_uri TEXT NOT NULL,
+          desired_fingerprint TEXT,
+          reason TEXT NOT NULL,
+          attempt_count INTEGER NOT NULL DEFAULT 0,
+          next_attempt_at TEXT NOT NULL,
+          lease_owner TEXT,
+          lease_expires_at TEXT,
+          last_failure_category TEXT,
+          updated_at TEXT NOT NULL
+        )
+      `,
+        ]);
+    }
+    async getReconciliation(sourceKey) {
+        const result = await (0, db_1.executeQuery)(this.db, (0, drizzle_orm_1.sql) `
+      SELECT
+        source_key,
+        source_uri,
+        desired_fingerprint,
+        reason,
+        attempt_count,
+        next_attempt_at,
+        lease_owner,
+        lease_expires_at,
+        last_failure_category,
+        updated_at
+      FROM reader_reconciliation
+      WHERE source_key = ${sourceKey}
+      LIMIT 1
+    `);
+        return result.rows[0] ? mapReconciliationRow(result.rows[0]) : undefined;
+    }
+}
+exports.ReaderMaterializationRepository = ReaderMaterializationRepository;
+function computeReaderMaterializationFingerprint(input) {
+    const canonical = [
+        input.sourceKey,
+        input.sourceHash,
+        input.mediaType,
+        input.readerEngine,
+        input.readerVersion,
+        input.modelUri ?? 'no-model',
+        input.readerOptionsHash,
+    ].join('\0');
+    return `sha256:${(0, node_crypto_1.createHash)('sha256').update(canonical).digest('hex')}`;
+}
+function mapBodyRow(row) {
+    return {
+        fingerprint: row.fingerprint,
+        sourceKey: row.source_key,
+        sourceUri: row.source_uri,
+        sourceHash: row.source_hash,
+        mediaType: row.media_type,
+        readerEngine: row.reader_engine,
+        readerVersion: row.reader_version,
+        modelUri: row.model_uri ?? undefined,
+        readerOptionsHash: row.reader_options_hash,
+        representationHash: row.representation_hash,
+        markdown: row.markdown,
+        createdAt: parseTimestamp(row.created_at),
+    };
+}
+function mapReconciliationRow(row) {
+    return {
+        sourceKey: row.source_key,
+        sourceUri: row.source_uri,
+        desiredFingerprint: row.desired_fingerprint ?? undefined,
+        reason: row.reason,
+        attemptCount: Number(row.attempt_count),
+        nextAttemptAt: parseTimestamp(row.next_attempt_at),
+        leaseOwner: row.lease_owner ?? undefined,
+        leaseExpiresAt: row.lease_expires_at ? parseTimestamp(row.lease_expires_at) : undefined,
+        lastFailureCategory: row.last_failure_category ?? undefined,
+        updatedAt: parseTimestamp(row.updated_at),
+    };
+}
+function isIdempotentBody(existing, input) {
+    return existing.sourceKey === input.sourceKey &&
+        existing.sourceUri === input.sourceUri &&
+        existing.sourceHash === input.sourceHash &&
+        existing.mediaType === input.mediaType &&
+        existing.readerEngine === input.readerEngine &&
+        existing.readerVersion === input.readerVersion &&
+        (existing.modelUri ?? undefined) === (input.modelUri ?? undefined) &&
+        existing.readerOptionsHash === input.readerOptionsHash &&
+        existing.representationHash === input.representationHash &&
+        Buffer.from(existing.markdown).equals(Buffer.from(input.markdown));
+}
+function timestampText(db, date) {
+    const value = (0, db_1.toDbTimestamp)(db, date);
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    if (typeof value === 'number') {
+        return new Date(value * 1000).toISOString();
+    }
+    return date.toISOString();
+}
+function parseTimestamp(value) {
+    return (0, db_1.fromDbTimestamp)(value) ?? new Date(value);
+}
+//# sourceMappingURL=ReaderMaterializationRepository.js.map

package/dist/document/ReaderMaterializationRepository.jsonld

@@ -0,0 +1,84 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@undefineds.co/xpod/^0.0.0/components/context.jsonld"
+  ],
+  "@id": "npmd:@undefineds.co/xpod",
+  "components": [
+    {
+      "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository",
+      "@type": "Class",
+      "requireElement": "ReaderMaterializationRepository",
+      "parameters": [
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository_db",
+          "range": {
+            "@type": "ParameterRangeWildcard"
+          }
+        }
+      ],
+      "memberFields": [
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_db",
+          "memberFieldName": "db"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_ready",
+          "memberFieldName": "ready"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_constructor",
+          "memberFieldName": "constructor"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_putBody",
+          "memberFieldName": "putBody"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_getBody",
+          "memberFieldName": "getBody"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_enqueue",
+          "memberFieldName": "enqueue"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_listRunnable",
+          "memberFieldName": "listRunnable"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_claimNext",
+          "memberFieldName": "claimNext"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_fail",
+          "memberFieldName": "fail"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_complete",
+          "memberFieldName": "complete"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_releaseExpiredLeases",
+          "memberFieldName": "releaseExpiredLeases"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_moveSource",
+          "memberFieldName": "moveSource"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_initialize",
+          "memberFieldName": "initialize"
+        },
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository__member_getReconciliation",
+          "memberFieldName": "getReconciliation"
+        }
+      ],
+      "constructorArguments": [
+        {
+          "@id": "undefineds:dist/document/ReaderMaterializationRepository.jsonld#ReaderMaterializationRepository_db"
+        }
+      ]
+    }
+  ]
+}

package/dist/document/index.d.ts

@@ -4,8 +4,10 @@
 export * from './DocumentReader';
 export * from './Chunker';
 export * from './JinaReader';
-export * from './HeadingChunker';
+export * from './MarkdownRetrievalPointProjector';
 export * from './ReaderPolicy';
 export * from './ReaderAiConfig';
 export * from './L0SourceSummary';
 export * from './PaddleOcrReader';
+export * from './ReaderMaterialization';
+export * from './ReaderMaterializationRepository';

package/dist/document/index.js

@@ -22,9 +22,11 @@ __exportStar(require("./DocumentReader"), exports);
 __exportStar(require("./Chunker"), exports);
 // 实现
 __exportStar(require("./JinaReader"), exports);
-__exportStar(require("./HeadingChunker"), exports);
+__exportStar(require("./MarkdownRetrievalPointProjector"), exports);
 __exportStar(require("./ReaderPolicy"), exports);
 __exportStar(require("./ReaderAiConfig"), exports);
 __exportStar(require("./L0SourceSummary"), exports);
 __exportStar(require("./PaddleOcrReader"), exports);
+__exportStar(require("./ReaderMaterialization"), exports);
+__exportStar(require("./ReaderMaterializationRepository"), exports);
 //# sourceMappingURL=index.js.map