@dreamlake/dreamdb 0.1.0 → 0.1.1

package/dist/index.d.cts

@@ -4,6 +4,10 @@ type MultihashBytes = Uint8Array;
 type Base32Hash = string;
 /** Nanoseconds since UNIX epoch — the join key across all modalities. */
 type TimeAnchor = number;
+/** The spatial key for non-spatial data (images, video, JSON, scalars).
+ *  8 zero bytes in hex — used as the path segment between modality and hash
+ *  for any track that isn't spatially indexed by an embedding vector. */
+declare const ZERO_SPATIAL_KEY = "0000000000000000";
 type FieldKind = "image" | "audio" | "video" | "embedding" | "scalar";
 type ScalarValueType = "categorical" | "bool" | "int" | "float" | "string" | "timestamp";
 interface ImageFieldDef {
@@ -495,4 +499,4 @@ declare class Dataset {
     private _resolveScalarLabels;
 }
 
-export { type AudioFieldDef, type Backend, type BackendGetOptions, type BackendPutOptions, type Base32Hash, type Batch, type BucketRecord, type CborValue, type ChaCha20Stream, Dataset, type DatasetInfo, type EmbeddingFieldDef, type FieldDef, type FieldKind, type HistoryEntry, type ImageFieldDef, MULTIHASH_BYTES, type Manifest, type ManifestTrack, MemoryBackend, type MultihashBytes, type ParsedSpaceUri, type QueryStats, type RegistryEntry, type ResolvedTrack, S3Backend, SCALAR_ENCODING_MAP, type Sample, type SampleValue, type ScalarFieldDef, type ScalarValueType, Schema, type SnapshotInfo, Space, type SpaceOptions, type SpatialDispatcher, type TimeAnchor, type VectorCompressorDecoder, type VectorQueryOptions, type VectorQueryResult, type VideoFieldDef, type WhereClause, base32ToBytes, bitsFromProjections, bitsToBase2, buildPqCosineDecoder, buildRabitqCosineDecoder, bytesToBase32, chacha20Stream, decodeBucketAnchorsOnly, decodeBucketRecords, decodeCbor, deriveHyperplanesLshCosine, deriveRotationMatrix, dotF32, dotL2Normalized, encodeCbor, modalityForField, multiProbeKeys, normalizeL2, normalizeL2F32, parseSpaceUri, peekBucketHeader, projectionsLshCosine, scalarModalityFragment };
+export { type AudioFieldDef, type Backend, type BackendGetOptions, type BackendPutOptions, type Base32Hash, type Batch, type BucketRecord, type CborValue, type ChaCha20Stream, Dataset, type DatasetInfo, type EmbeddingFieldDef, type FieldDef, type FieldKind, type HistoryEntry, type ImageFieldDef, MULTIHASH_BYTES, type Manifest, type ManifestTrack, MemoryBackend, type MultihashBytes, type ParsedSpaceUri, type QueryStats, type RegistryEntry, type ResolvedTrack, S3Backend, SCALAR_ENCODING_MAP, type Sample, type SampleValue, type ScalarFieldDef, type ScalarValueType, Schema, type SnapshotInfo, Space, type SpaceOptions, type SpatialDispatcher, type TimeAnchor, type VectorCompressorDecoder, type VectorQueryOptions, type VectorQueryResult, type VideoFieldDef, type WhereClause, ZERO_SPATIAL_KEY, base32ToBytes, bitsFromProjections, bitsToBase2, buildPqCosineDecoder, buildRabitqCosineDecoder, bytesToBase32, chacha20Stream, decodeBucketAnchorsOnly, decodeBucketRecords, decodeCbor, deriveHyperplanesLshCosine, deriveRotationMatrix, dotF32, dotL2Normalized, encodeCbor, modalityForField, multiProbeKeys, normalizeL2, normalizeL2F32, parseSpaceUri, peekBucketHeader, projectionsLshCosine, scalarModalityFragment };

package/dist/index.d.ts

@@ -4,6 +4,10 @@ type MultihashBytes = Uint8Array;
 type Base32Hash = string;
 /** Nanoseconds since UNIX epoch — the join key across all modalities. */
 type TimeAnchor = number;
+/** The spatial key for non-spatial data (images, video, JSON, scalars).
+ *  8 zero bytes in hex — used as the path segment between modality and hash
+ *  for any track that isn't spatially indexed by an embedding vector. */
+declare const ZERO_SPATIAL_KEY = "0000000000000000";
 type FieldKind = "image" | "audio" | "video" | "embedding" | "scalar";
 type ScalarValueType = "categorical" | "bool" | "int" | "float" | "string" | "timestamp";
 interface ImageFieldDef {
@@ -495,4 +499,4 @@ declare class Dataset {
     private _resolveScalarLabels;
 }
 
-export { type AudioFieldDef, type Backend, type BackendGetOptions, type BackendPutOptions, type Base32Hash, type Batch, type BucketRecord, type CborValue, type ChaCha20Stream, Dataset, type DatasetInfo, type EmbeddingFieldDef, type FieldDef, type FieldKind, type HistoryEntry, type ImageFieldDef, MULTIHASH_BYTES, type Manifest, type ManifestTrack, MemoryBackend, type MultihashBytes, type ParsedSpaceUri, type QueryStats, type RegistryEntry, type ResolvedTrack, S3Backend, SCALAR_ENCODING_MAP, type Sample, type SampleValue, type ScalarFieldDef, type ScalarValueType, Schema, type SnapshotInfo, Space, type SpaceOptions, type SpatialDispatcher, type TimeAnchor, type VectorCompressorDecoder, type VectorQueryOptions, type VectorQueryResult, type VideoFieldDef, type WhereClause, base32ToBytes, bitsFromProjections, bitsToBase2, buildPqCosineDecoder, buildRabitqCosineDecoder, bytesToBase32, chacha20Stream, decodeBucketAnchorsOnly, decodeBucketRecords, decodeCbor, deriveHyperplanesLshCosine, deriveRotationMatrix, dotF32, dotL2Normalized, encodeCbor, modalityForField, multiProbeKeys, normalizeL2, normalizeL2F32, parseSpaceUri, peekBucketHeader, projectionsLshCosine, scalarModalityFragment };
+export { type AudioFieldDef, type Backend, type BackendGetOptions, type BackendPutOptions, type Base32Hash, type Batch, type BucketRecord, type CborValue, type ChaCha20Stream, Dataset, type DatasetInfo, type EmbeddingFieldDef, type FieldDef, type FieldKind, type HistoryEntry, type ImageFieldDef, MULTIHASH_BYTES, type Manifest, type ManifestTrack, MemoryBackend, type MultihashBytes, type ParsedSpaceUri, type QueryStats, type RegistryEntry, type ResolvedTrack, S3Backend, SCALAR_ENCODING_MAP, type Sample, type SampleValue, type ScalarFieldDef, type ScalarValueType, Schema, type SnapshotInfo, Space, type SpaceOptions, type SpatialDispatcher, type TimeAnchor, type VectorCompressorDecoder, type VectorQueryOptions, type VectorQueryResult, type VideoFieldDef, type WhereClause, ZERO_SPATIAL_KEY, base32ToBytes, bitsFromProjections, bitsToBase2, buildPqCosineDecoder, buildRabitqCosineDecoder, bytesToBase32, chacha20Stream, decodeBucketAnchorsOnly, decodeBucketRecords, decodeCbor, deriveHyperplanesLshCosine, deriveRotationMatrix, dotF32, dotL2Normalized, encodeCbor, modalityForField, multiProbeKeys, normalizeL2, normalizeL2F32, parseSpaceUri, peekBucketHeader, projectionsLshCosine, scalarModalityFragment };

package/dist/index.js

@@ -321,16 +321,33 @@ var S3Backend = class {
     return path;
   }
   async list(prefix) {
-    const url = `${this.baseUrl}?list-type=2&prefix=${encodeURIComponent(prefix)}`;
-    const r = await fetch(url);
-    if (!r.ok) throw new Error(`LIST ${url}: HTTP ${r.status}`);
-    const text = await r.text();
     const keys = [];
-    const regex = /<Key>([^<]+)<\/Key>/g;
-    let match;
-    while ((match = regex.exec(text)) !== null) {
-      keys.push(match[1]);
-    }
+    let continuationToken;
+    do {
+      let url = `${this.baseUrl}?list-type=2&prefix=${encodeURIComponent(prefix)}`;
+      if (continuationToken) {
+        url += `&continuation-token=${encodeURIComponent(continuationToken)}`;
+      }
+      const r = await fetch(url);
+      if (!r.ok) throw new Error(`LIST ${url}: HTTP ${r.status}`);
+      const text = await r.text();
+      const regex = /<Key>([^<]+)<\/Key>/g;
+      let match;
+      while ((match = regex.exec(text)) !== null) {
+        keys.push(match[1]);
+      }
+      const truncatedMatch = text.match(/<IsTruncated>(\w+)<\/IsTruncated>/);
+      const isTruncated = truncatedMatch?.[1] === "true";
+      if (isTruncated) {
+        const tokenMatch = text.match(
+          /<NextContinuationToken>([^<]+)<\/NextContinuationToken>/
+        );
+        continuationToken = tokenMatch?.[1];
+        if (!continuationToken) break;
+      } else {
+        continuationToken = void 0;
+      }
+    } while (continuationToken);
     return keys;
   }
   async delete(path) {
@@ -1310,6 +1327,17 @@ var Space = class _Space {
   }
 };
 
+// src/types.ts
+var ZERO_SPATIAL_KEY = "0000000000000000";
+var SCALAR_ENCODING_MAP = {
+  categorical: "cat",
+  bool: "bool",
+  int: "i64",
+  float: "f64",
+  string: "str",
+  timestamp: "ts"
+};
+
 // src/dataset.ts
 var Schema = class {
   fields = [];
@@ -1619,7 +1647,7 @@ var Dataset = class _Dataset {
           const fragHash = contentHash(data);
           const fragHashB32 = bytesToBase32(fragHash);
           await this.backend.put(
-            `${this._timelineHash}/${modality}/0000000000000000/${fragHashB32}`,
+            `${this._timelineHash}/${modality}/${ZERO_SPATIAL_KEY}/${fragHashB32}`,
             data
           );
           newEntries.push([
@@ -1867,7 +1895,7 @@ var Dataset = class _Dataset {
         const fragHash = bytesToBase32(e[3]);
         try {
           const fragBytes = await this.backend.get(
-            `${this._timelineHash}/${bfMod}/0000000000000000/${fragHash}`
+            `${this._timelineHash}/${bfMod}/${ZERO_SPATIAL_KEY}/${fragHash}`
           );
           fieldData.set(tStart, fragBytes);
         } catch {
@@ -2170,16 +2198,6 @@ function schemaFromCbor(obj) {
   }
   return schema;
 }
-
-// src/types.ts
-var SCALAR_ENCODING_MAP = {
-  categorical: "cat",
-  bool: "bool",
-  int: "i64",
-  float: "f64",
-  string: "str",
-  timestamp: "ts"
-};
 export {
   Dataset,
   MULTIHASH_BYTES,
@@ -2188,6 +2206,7 @@ export {
   SCALAR_ENCODING_MAP,
   Schema,
   Space,
+  ZERO_SPATIAL_KEY,
   base32ToBytes,
   bitsFromProjections,
   bitsToBase2,