@noy-db/hub 0.2.0-pre.19 → 0.2.0-pre.20

package/dist/attestation/index.d.cts

@@ -1,13 +1,13 @@
-import { aP as NoydbStore } from '../types-D-gr5t0G.cjs';
+import { aW as NoydbStore } from '../types-DyOI6XZ_.cjs';
 import { AttestationFieldSchema, QrPayload, RevocationList } from '@noy-db/attestation';
 export { AttestationFieldSchema, QrPayload, RevocationList, VerifyInput, VerifyResult, decodeQr, isRevoked, signRevocationList, verifyAttestation, verifyRevocationList } from '@noy-db/attestation';
-export { s as AttestationError } from '../errors-DL-zTrrF.cjs';
+export { s as AttestationError } from '../errors-Dkc_fi-S.cjs';
 import '../lazy-builder-eYZzLEL1.cjs';
 import '../predicate-BmhBSPCH.cjs';
-import '../strategy-C5ol6NdV.cjs';
+import '../strategy-WtB-jXYv.cjs';
 import '../strategy-BSxFXGzb.cjs';
 import '../index-BMmajblo.cjs';
-import '../index-BM7O48Ur.cjs';
+import '../index-tZqVB9g5.cjs';
 
 /** Everything issueAttestationCore needs from the Vault, injected for testability. */
 interface IssueContext {

package/dist/attestation/index.d.ts

@@ -1,13 +1,13 @@
-import { aP as NoydbStore } from '../types-CraiZOyO.js';
+import { aW as NoydbStore } from '../types-DLfWFr6U.js';
 import { AttestationFieldSchema, QrPayload, RevocationList } from '@noy-db/attestation';
 export { AttestationFieldSchema, QrPayload, RevocationList, VerifyInput, VerifyResult, decodeQr, isRevoked, signRevocationList, verifyAttestation, verifyRevocationList } from '@noy-db/attestation';
-export { s as AttestationError } from '../errors-DL-zTrrF.js';
+export { s as AttestationError } from '../errors-Dkc_fi-S.js';
 import '../lazy-builder-ChSqcF5t.js';
 import '../predicate-BmhBSPCH.js';
-import '../strategy-BDxQnnTX.js';
+import '../strategy-54eIwox5.js';
 import '../strategy-BSxFXGzb.js';
 import '../index-BMmajblo.js';
-import '../index-BelbyUwz.js';
+import '../index-Bm9hIY7t.js';
 
 /** Everything issueAttestationCore needs from the Vault, injected for testability. */
 interface IssueContext {

package/dist/attestation/index.js

@@ -1,21 +1,21 @@
 import {
   issueAttestationCore
-} from "../chunk-Y5J63SMF.js";
+} from "../chunk-F5ILTHMU.js";
 import {
   getRevokedDocIdsCore,
   publishRevocationListCore,
   revokeDocCore,
   unrevokeDocCore
-} from "../chunk-5IGWRMEC.js";
+} from "../chunk-IVZWHIEK.js";
 import {
   ATTESTATIONS_COLLECTION
-} from "../chunk-V5FZWQNN.js";
+} from "../chunk-TAMRU7A2.js";
 import "../chunk-FZU343FL.js";
-import "../chunk-SHIUFIPW.js";
-import "../chunk-OKOKPYWH.js";
+import "../chunk-TA6HPKWQ.js";
+import "../chunk-37VGJM3T.js";
 import {
   AttestationError
-} from "../chunk-DJF3FXW5.js";
+} from "../chunk-OTWT6BAJ.js";
 
 // src/attestation/index.ts
 import { verifyAttestation, decodeQr, verifyRevocationList, isRevoked, signRevocationList } from "@noy-db/attestation";

package/dist/blobs/index.cjs

@@ -397,7 +397,9 @@ __export(blobs_exports, {
   createExportBlobsHandle: () => createExportBlobsHandle,
   detectMagic: () => detectMagic,
   detectMimeType: () => detectMimeType,
+  importExternalObjects: () => importExternalObjects,
   isPreCompressed: () => isPreCompressed,
+  memoryObjectProjection: () => memoryObjectProjection,
   runCompaction: () => runCompaction,
   withBlobs: () => withBlobs
 });
@@ -677,6 +679,9 @@ var BlobSet = class {
   userId;
   maxBlobBytes;
   erasableBlobs;
+  debugPlaintext;
+  objectStore;
+  blobFields;
   constructor(opts) {
     this.store = opts.store;
     this.vault = opts.vault;
@@ -687,6 +692,9 @@ var BlobSet = class {
     this.userId = opts.userId;
     this.maxBlobBytes = opts.maxBlobBytes;
     this.erasableBlobs = opts.erasableBlobs === true;
+    this.debugPlaintext = opts.debugPlaintext === true;
+    this.objectStore = opts.objectStore;
+    this.blobFields = opts.blobFields;
   }
   /**
    * Resolve the key the blob's CHUNKS are encrypted under.
@@ -1074,6 +1082,49 @@ var BlobSet = class {
    *    If overwriting an existing slot, decrements the old eTag's refCount.
    */
   async put(slotName, data, opts) {
+    if (this.objectStore && this.blobFields?.[slotName]?.external) {
+      const policy = this.blobFields[slotName];
+      let contentType = opts?.mimeType;
+      if (!contentType) {
+        const detected = detectMagic(data.subarray(0, 16));
+        contentType = detected?.mime ?? "application/octet-stream";
+      }
+      const key = `${this.collection}/${this.recordId}/${slotName}`;
+      const isPublic = policy.public === true;
+      const backlink = await this.buildBacklink(slotName, policy.backlink ?? "opaque-token");
+      await this.objectStore.putObject(key, data, {
+        contentType,
+        public: isPublic,
+        ...backlink.userMeta ? { userMeta: backlink.userMeta } : {}
+      });
+      const uploaderUserId2 = opts?.uploadedBy ?? this.userId;
+      let oldETag;
+      await this.casUpdateSlots((slots) => {
+        oldETag = slots[slotName]?.eTag || void 0;
+        const prevMeta = slots[slotName]?.external?.meta;
+        slots[slotName] = {
+          eTag: "",
+          external: {
+            key,
+            contentType,
+            ...isPublic ? { public: true } : {},
+            ...backlink.token ? { backlink: backlink.token } : {},
+            ...prevMeta ? { meta: prevMeta } : {}
+          },
+          filename: slotName,
+          size: data.byteLength,
+          mimeType: contentType,
+          uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          ...uploaderUserId2 !== void 0 ? { uploadedBy: uploaderUserId2 } : {}
+        };
+        return slots;
+      });
+      if (oldETag) {
+        await this.releaseRef(oldETag, 1, false).catch(() => {
+        });
+      }
+      return;
+    }
     const blobDEK = this.encrypted ? await this.getDEK(BLOB_COLLECTION) : null;
     const eTag = blobDEK ? await hmacSha256Hex(blobDEK, data) : await plainSha256Hex(data);
     let mimeType = opts?.mimeType;
@@ -1089,12 +1140,13 @@ var BlobSet = class {
     } else {
       shouldCompress = true;
     }
+    if (this.debugPlaintext) shouldCompress = false;
     const existingBlob = await this.loadBlobObject(eTag);
     if (existingBlob) {
       await this.casUpdateRefCount(eTag, 1);
     } else {
       const { bytes: compressed, algorithm } = shouldCompress ? await compressBytes(data) : { bytes: data, algorithm: "none" };
-      const chunkSize = this.effectiveChunkSize(opts);
+      const chunkSize = this.debugPlaintext ? Math.max(compressed.byteLength, 1) : this.effectiveChunkSize(opts);
       const chunkCount = Math.max(1, Math.ceil(compressed.byteLength / chunkSize));
       let chunkKey = blobDEK;
       let wrappedCek;
@@ -1159,10 +1211,114 @@ var BlobSet = class {
     const { slots } = await this.loadSlots();
     const slot = slots[slotName];
     if (!slot) return null;
+    if (slot.external) {
+      if (!this.objectStore) {
+        throw new NotFoundError(`Blob slot "${slotName}" is external but no objectStore is configured`);
+      }
+      return this.objectStore.getObject(slot.external.key);
+    }
     const result = await this.loadBlobObject(slot.eTag);
     if (!result) return null;
     return this.fetchAllChunks(result.blob);
   }
+  /**
+   * A URL to fetch an `external` slot's object directly — presigned
+   * (time-limited) or public, per the projection. Returns `null` if the slot
+   * does not exist. Throws for a non-external slot (use `get()`/`response()`).
+   */
+  async url(slotName, opts) {
+    const { slots } = await this.loadSlots();
+    const slot = slots[slotName];
+    if (!slot) return null;
+    if (!slot.external) {
+      throw new NotFoundError(`Blob slot "${slotName}" is not external \u2014 url() is only for external fields`);
+    }
+    if (!this.objectStore) {
+      throw new NotFoundError(`Blob slot "${slotName}" is external but no objectStore is configured`);
+    }
+    return this.objectStore.objectUrl(slot.external.key, opts);
+  }
+  /**
+   * Build the backlink stamped onto an external object's metadata — the
+   * self-describing "secondary store" reference back to this record. See
+   * {@link BlobFieldPolicy.backlink}. Returns the `userMeta` to attach and, for
+   * `opaque-token`, the `token` to record on the slot.
+   */
+  async buildBacklink(slotName, mode) {
+    if (mode === "none") return {};
+    const ref = { vault: this.vault, collection: this.collection, record: this.recordId, field: slotName };
+    if (mode === "plain") {
+      return {
+        userMeta: {
+          "noydb-vault": ref.vault,
+          "noydb-collection": ref.collection,
+          "noydb-record": ref.record,
+          "noydb-field": ref.field
+        }
+      };
+    }
+    if (mode === "encrypted" && this.encrypted) {
+      const dek = await this.getDEK(BLOB_COLLECTION);
+      const { iv, data } = await encrypt(JSON.stringify(ref), dek);
+      return { userMeta: { "noydb-backlink-enc": `${iv}.${data}` } };
+    }
+    const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
+    const token = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
+    return { userMeta: { "noydb-backlink": token }, token };
+  }
+  /**
+   * Adopt an EXISTING object in the projection into this slot **without
+   * re-uploading** — used by import/bootstrap to anchor objects already in the
+   * bucket. Writes the external slot record (the catalog entry).
+   */
+  async adoptExternal(slotName, ref) {
+    const uploaderUserId = this.userId;
+    await this.casUpdateSlots((slots) => {
+      slots[slotName] = {
+        eTag: "",
+        external: {
+          key: ref.key,
+          ...ref.contentType ? { contentType: ref.contentType } : {},
+          ...ref.public ? { public: true } : {},
+          ...ref.backlink ? { backlink: ref.backlink } : {},
+          ...ref.meta ? { meta: ref.meta } : {}
+        },
+        filename: slotName,
+        size: ref.size ?? 0,
+        ...ref.contentType ? { mimeType: ref.contentType } : {},
+        uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        ...uploaderUserId !== void 0 ? { uploadedBy: uploaderUserId } : {}
+      };
+      return slots;
+    });
+  }
+  /**
+   * Merge derived metadata (video `duration`, image `width`/`height`, arbitrary
+   * metatags) into an external slot's secondary metadata store. Typically called
+   * from an AWS-side processing callback (MediaConvert / ffprobe / Rekognition)
+   * once it has probed the object. No-op for a missing or non-external slot.
+   */
+  async setExternalMeta(slotName, meta) {
+    await this.casUpdateSlots((slots) => {
+      const slot = slots[slotName];
+      if (!slot?.external) return null;
+      slots[slotName] = {
+        ...slot,
+        external: { ...slot.external, meta: { ...slot.external.meta, ...meta } }
+      };
+      return slots;
+    });
+  }
+  /**
+   * Read an external slot's synced derived metadata (the secondary store).
+   * Returns `null` for a missing or non-external slot, `{}` if none synced yet.
+   */
+  async externalMeta(slotName) {
+    const { slots } = await this.loadSlots();
+    const slot = slots[slotName];
+    if (!slot?.external) return null;
+    return slot.external.meta ?? {};
+  }
   /**
    * List all slot entries for this record.
    * Returns metadata only — no chunk data is loaded.
@@ -1177,12 +1333,19 @@ var BlobSet = class {
    */
   async delete(slotName) {
     let eTagToRelease;
+    let externalKeyToDelete;
     await this.casUpdateSlots((slots) => {
       if (!(slotName in slots)) return null;
-      eTagToRelease = slots[slotName].eTag;
+      const slot = slots[slotName];
+      if (slot.external) externalKeyToDelete = slot.external.key;
+      else eTagToRelease = slot.eTag;
       delete slots[slotName];
       return slots;
     });
+    if (externalKeyToDelete && this.objectStore) {
+      await this.objectStore.deleteObject(externalKeyToDelete).catch(() => {
+      });
+    }
     if (eTagToRelease) {
       await this.releaseRef(eTagToRelease, 1, false).catch(() => {
       });
@@ -1449,6 +1612,91 @@ function withBlobs() {
   };
 }
 
+// src/blobs/object-projection.ts
+function memoryObjectProjection(opts = {}) {
+  const base = opts.baseUrl ?? "memory://objects";
+  const store = /* @__PURE__ */ new Map();
+  return {
+    name: "memory",
+    async putObject(key, bytes, o) {
+      store.set(key, {
+        bytes,
+        contentType: o.contentType,
+        public: o.public === true,
+        ...o.userMeta ? { userMeta: o.userMeta } : {}
+      });
+    },
+    async getObject(key) {
+      return store.get(key)?.bytes ?? null;
+    },
+    async deleteObject(key) {
+      store.delete(key);
+    },
+    async headObject(key) {
+      const e = store.get(key);
+      if (!e) return null;
+      return {
+        size: e.bytes.byteLength,
+        contentType: e.contentType,
+        ...e.userMeta ? { userMeta: e.userMeta } : {}
+      };
+    },
+    async objectUrl(key, o) {
+      const e = store.get(key);
+      if (e?.public) return `${base}/${key}`;
+      return `${base}/${key}?sig=memory&expires=${o?.expiresInSeconds ?? 900}`;
+    },
+    async putUrl(key, o) {
+      return `${base}/${key}?upload=memory&ct=${encodeURIComponent(o.contentType)}`;
+    },
+    async listPrefix(prefix) {
+      const out = [];
+      for (const [key, e] of store) {
+        if (!key.startsWith(prefix)) continue;
+        out.push({
+          key,
+          meta: { size: e.bytes.byteLength, contentType: e.contentType, ...e.userMeta ? { userMeta: e.userMeta } : {} }
+        });
+      }
+      return out;
+    }
+  };
+}
+
+// src/blobs/import-external.ts
+function defaultDeriveRecordId(key) {
+  const parts = key.split("/");
+  return parts.length >= 2 ? parts[parts.length - 2] ?? null : null;
+}
+async function importExternalObjects(args) {
+  const { collection, objectStore, field } = args;
+  const opts = args.options ?? {};
+  const derive = opts.deriveRecordId ?? defaultDeriveRecordId;
+  const makeRecord = opts.makeRecord ?? ((id) => ({ id }));
+  const objects = await objectStore.listPrefix(opts.prefix ?? "");
+  const recordIds = [];
+  let imported = 0;
+  let skipped = 0;
+  for (const { key, meta } of objects) {
+    const recordId = derive(key);
+    if (!recordId) {
+      skipped++;
+      continue;
+    }
+    if (await collection.get(recordId) == null) {
+      await collection.put(recordId, makeRecord(recordId));
+    }
+    await collection.blob(recordId).adoptExternal(field, {
+      key,
+      ...meta.size !== void 0 ? { size: meta.size } : {},
+      ...meta.contentType ? { contentType: meta.contentType } : {}
+    });
+    recordIds.push(recordId);
+    imported++;
+  }
+  return { imported, skipped, recordIds };
+}
+
 // src/blobs/blob-compaction.ts
 init_crypto();
 var BLOB_EVICTION_AUDIT_COLLECTION = "_blob_eviction_audit";
@@ -1710,7 +1958,9 @@ function generateBatchId() {
   createExportBlobsHandle,
   detectMagic,
   detectMimeType,
+  importExternalObjects,
   isPreCompressed,
+  memoryObjectProjection,
   runCompaction,
   withBlobs
 });