@pasko70/pibo 3.2.4 → 3.2.5

package/dist/data/payload-store.js

@@ -34,11 +34,8 @@ export class PayloadStore {
         const createdAt = input.createdAt ?? new Date().toISOString();
         const bytes = payloadToBytes(input.value, contentType);
         const sha256 = createHash("sha256").update(bytes).digest("hex");
-        const existing = this.findBySha256(sha256);
+        const existing = this.findByIdentity(sha256, contentType, input.retentionClass);
         if (existing) {
-            if (existing.contentType !== contentType || existing.retentionClass !== input.retentionClass) {
-                throw new PiboPayloadMetadataConflictError(sha256, existing.contentType, contentType, existing.retentionClass, input.retentionClass);
-            }
             this.db.prepare("UPDATE payloads SET ref_count = ref_count + 1 WHERE id = ?").run(existing.id);
             return this.getPayload(existing.id) ?? existing;
         }
@@ -46,7 +43,7 @@ export class PayloadStore {
         const encoding = shouldCompress ? "gzip" : "identity";
         const bytesToStore = shouldCompress ? gzipSync(bytes) : bytes;
         const compressedByteSize = shouldCompress ? bytesToStore.byteLength : null;
-        const relativePath = buildRelativePayloadPath(sha256, contentType, encoding);
+        const relativePath = buildRelativePayloadPath(sha256, contentType, input.retentionClass, encoding);
         const absolutePath = this.rootDir === ":memory:" ? relativePath : join(this.rootDir, relativePath);
         writePayloadFile(absolutePath, bytesToStore);
         const id = input.id ?? `payload_${randomUUID()}`;
@@ -127,7 +124,14 @@ export class PayloadStore {
         return JSON.parse(Buffer.from(this.readPayloadBytesBounded(id, maxBytes)).toString("utf8"));
     }
     findBySha256(sha256) {
-        const row = this.db.prepare("SELECT * FROM payloads WHERE sha256 = ?").get(sha256);
+        const row = this.db.prepare("SELECT * FROM payloads WHERE sha256 = ? ORDER BY created_at ASC, id ASC LIMIT 1").get(sha256);
+        return row ? payloadFromRow(row) : undefined;
+    }
+    findByIdentity(sha256, contentType, retentionClass) {
+        const row = this.db.prepare(`
+			SELECT * FROM payloads
+			WHERE sha256 = ? AND content_type = ? AND retention_class = ?
+		`).get(sha256, contentType, retentionClass);
         return row ? payloadFromRow(row) : undefined;
     }
     releaseReferences(id, count = 1) {
@@ -185,12 +189,12 @@ function defaultContentType(value) {
     return "application/json";
 }
 function payloadToBytes(value, contentType) {
-    if (typeof value === "string")
-        return Buffer.from(value, "utf8");
     if (value instanceof Uint8Array)
         return value;
     if (contentType.includes("json"))
         return Buffer.from(JSON.stringify(value), "utf8");
+    if (typeof value === "string")
+        return Buffer.from(value, "utf8");
     return Buffer.from(String(value), "utf8");
 }
 function previewTextFromValue(value) {
@@ -200,10 +204,16 @@ function previewTextFromValue(value) {
     const normalized = text.replace(/\s+/g, " ").trim();
     return normalized ? normalized.slice(0, 1024) : undefined;
 }
-function buildRelativePayloadPath(sha256, contentType, encoding) {
+function buildRelativePayloadPath(sha256, contentType, retentionClass, encoding) {
     const extension = extensionForContentType(contentType);
     const suffix = encoding === "gzip" ? `${extension}.gz` : extension;
-    return join("sha256", sha256.slice(0, 2), sha256.slice(2, 4), `${sha256}.${suffix}`);
+    const metadataHash = createHash("sha256")
+        .update(contentType)
+        .update("\0")
+        .update(retentionClass)
+        .digest("hex")
+        .slice(0, 16);
+    return join("sha256", sha256.slice(0, 2), sha256.slice(2, 4), `${sha256}.${metadataHash}.${suffix}`);
 }
 function extensionForContentType(contentType) {
     if (contentType.includes("json"))

package/dist/data/schema.js

@@ -1,6 +1,27 @@
-export const PIBO_DATA_SCHEMA_VERSION = 7;
+export const PIBO_DATA_SCHEMA_VERSION = 8;
 const NATIVE_HISTORY_FALLBACK_SCHEMA_VERSION = 5;
 const retiredScopeColumn = ["owner", "scope"].join("_");
+const payloadTableDefinition = `
+	id TEXT PRIMARY KEY,
+	sha256 TEXT NOT NULL,
+	storage_kind TEXT NOT NULL,
+	storage_path TEXT,
+	content_type TEXT NOT NULL,
+	encoding TEXT NOT NULL DEFAULT 'gzip',
+	byte_size INTEGER NOT NULL,
+	compressed_byte_size INTEGER,
+	preview_text TEXT,
+	retention_class TEXT NOT NULL,
+	ref_count INTEGER NOT NULL DEFAULT 0,
+	status TEXT NOT NULL DEFAULT 'committed',
+	created_at TEXT NOT NULL,
+	last_verified_at TEXT
+`;
+const payloadTableColumns = [
+    "id", "sha256", "storage_kind", "storage_path", "content_type", "encoding",
+    "byte_size", "compressed_byte_size", "preview_text", "retention_class", "ref_count",
+    "status", "created_at", "last_verified_at",
+];
 const retiredScopeTables = [
     {
         name: "sessions",
@@ -93,6 +114,26 @@ function rebuildRetiredScopeTables(db, tablesToRebuild) {
 		`);
     }
 }
+function payloadTableNeedsMetadataIdentityRebuild(db) {
+    const table = db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'payloads'").get();
+    if (!table)
+        return false;
+    const indexes = db.prepare("SELECT name FROM pragma_index_list('payloads') WHERE \"unique\" = 1").all();
+    return indexes.some((index) => {
+        const columns = db.prepare("SELECT name FROM pragma_index_info(?) ORDER BY seqno").all(index.name);
+        return columns.length === 1 && columns[0]?.name === "sha256";
+    });
+}
+function rebuildPayloadTableForMetadataIdentity(db) {
+    const columns = payloadTableColumns.join(", ");
+    db.exec(`
+		DROP TABLE IF EXISTS __pibo_schema_v8_payloads;
+		CREATE TABLE __pibo_schema_v8_payloads (${payloadTableDefinition});
+		INSERT INTO __pibo_schema_v8_payloads (${columns}) SELECT ${columns} FROM payloads;
+		DROP TABLE payloads;
+		ALTER TABLE __pibo_schema_v8_payloads RENAME TO payloads;
+	`);
+}
 export function assertSupportedPiboDataSchemaVersion(db) {
     const version = Number(db.prepare("PRAGMA user_version").get()?.user_version ?? 0);
     if (version > PIBO_DATA_SCHEMA_VERSION) {
@@ -101,6 +142,7 @@ export function assertSupportedPiboDataSchemaVersion(db) {
     return version;
 }
 export const PIBO_DATA_SCHEMA_MIGRATION_STEPS = [
+    "payload-metadata-identity",
     "schema",
     "runtime-bindings",
     "render-high-water",
@@ -114,20 +156,22 @@ export const PIBO_DATA_SCHEMA_MIGRATION_STEPS = [
 export function applyPiboDataSchema(db, hooks = {}) {
     const previousVersion = assertSupportedPiboDataSchemaVersion(db);
     const tablesToRebuild = retiredScopeTablesToRebuild(db);
+    const rebuildPayloadTable = payloadTableNeedsMetadataIdentityRebuild(db);
+    const requiresTableRebuild = tablesToRebuild.length > 0 || rebuildPayloadTable;
     const ownsTransaction = !db.isTransaction;
-    if (!ownsTransaction && tablesToRebuild.length > 0) {
+    if (!ownsTransaction && requiresTableRebuild) {
         throw new Error("Pibo data schema migration requires an independent transaction");
     }
     const foreignKeysEnabled = ownsTransaction
-        && tablesToRebuild.length > 0
+        && requiresTableRebuild
         && Number(db.prepare("PRAGMA foreign_keys").get().foreign_keys ?? 0) === 1;
     if (foreignKeysEnabled)
         db.exec("PRAGMA foreign_keys = OFF");
     try {
         if (ownsTransaction)
             db.exec("BEGIN IMMEDIATE");
-        applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRebuild);
-        if (tablesToRebuild.length > 0) {
+        applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRebuild, rebuildPayloadTable);
+        if (requiresTableRebuild) {
             const violations = db.prepare("PRAGMA foreign_key_check").all();
             if (violations.length > 0) {
                 throw new Error(`Pibo data schema migration would retain ${violations.length} foreign-key violation(s)`);
@@ -146,11 +190,14 @@ export function applyPiboDataSchema(db, hooks = {}) {
             db.exec("PRAGMA foreign_keys = ON");
     }
 }
-function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRebuild) {
+function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRebuild, rebuildPayloadTable) {
     const existingSessionCount = db.prepare("SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table' AND name = 'sessions'").get();
     const hadSessionsBeforeMigration = existingSessionCount.count > 0
         && Number(db.prepare("SELECT COUNT(*) AS count FROM sessions").get().count) > 0;
     rebuildRetiredScopeTables(db, tablesToRebuild);
+    if (rebuildPayloadTable)
+        rebuildPayloadTableForMetadataIdentity(db);
+    hooks.afterStep?.("payload-metadata-identity");
     db.exec(`
 		CREATE TABLE IF NOT EXISTS sessions (
 			id TEXT PRIMARY KEY,
@@ -300,22 +347,9 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
 		);
 
 
-		CREATE TABLE IF NOT EXISTS payloads (
-			id TEXT PRIMARY KEY,
-			sha256 TEXT NOT NULL UNIQUE,
-			storage_kind TEXT NOT NULL,
-			storage_path TEXT,
-			content_type TEXT NOT NULL,
-			encoding TEXT NOT NULL DEFAULT 'gzip',
-			byte_size INTEGER NOT NULL,
-			compressed_byte_size INTEGER,
-			preview_text TEXT,
-			retention_class TEXT NOT NULL,
-			ref_count INTEGER NOT NULL DEFAULT 0,
-			status TEXT NOT NULL DEFAULT 'committed',
-			created_at TEXT NOT NULL,
-			last_verified_at TEXT
-		);
+		CREATE TABLE IF NOT EXISTS payloads (${payloadTableDefinition});
+		CREATE UNIQUE INDEX IF NOT EXISTS idx_payloads_identity
+			ON payloads(sha256, content_type, retention_class);
 
 		CREATE TABLE IF NOT EXISTS event_log (
 			stream_id INTEGER PRIMARY KEY,

package/dist/tools/mcp-bridge.js

@@ -180,7 +180,7 @@ async function piboResultToMcp(result, options) {
                 tool: options.tool,
                 toolCallId: options.toolCallId,
                 contentType: "application/json",
-                value: encoded,
+                value: structuredContent,
             });
             payloadRefs.push(stored.ref);
             structuredContent = undefined;

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@pasko70/pibo",
-  "version": "3.2.4",
+  "version": "3.2.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@pasko70/pibo",
-      "version": "3.2.4",
+      "version": "3.2.5",
       "workspaces": [
         "packages/workflows"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pasko70/pibo",
-  "version": "3.2.4",
+  "version": "3.2.5",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",