@dreamlake/dreamdb 0.1.0 → 0.1.1

package/README.md

@@ -0,0 +1,62 @@
+# @dreamlake/dreamdb
+
+TypeScript driver for [DreamDB](https://dreamdb.dreamlake.ai) — a storage and retrieval protocol for multimodal signals anchored to a shared timeline.
+
+## Install
+
+```bash
+npm install @dreamlake/dreamdb
+```
+
+## Quick start
+
+```typescript
+import { Dataset, Schema, MemoryBackend } from '@dreamlake/dreamdb'
+
+// Define a schema
+const schema = new Schema()
+schema.addEmbedding('embedding', 16)
+schema.addImage('image', 'jpeg')
+schema.addScalarCategorical('label')
+
+// Create a dataset (in-memory for testing, or use S3Backend for production)
+const backend = new MemoryBackend()
+const ds = await Dataset.create('my-dataset', schema, backend)
+
+// Append samples
+await ds.appendMany([
+  { embedding: new Float32Array([...]), image: new Uint8Array([...]), label: 'cat' },
+  { embedding: new Float32Array([...]), image: new Uint8Array([...]), label: 'dog' },
+])
+
+// Vector query — find the 5 nearest neighbors
+const batches = await ds.iterVector({
+  field: 'embedding',
+  query: new Float32Array([...]),
+  topK: 5,
+})
+```
+
+## Browser usage
+
+```typescript
+import { S3Backend, Space } from '@dreamlake/dreamdb'
+
+const backend = new S3Backend('https://your-bucket.s3.amazonaws.com')
+const space = new Space(backend)
+
+// Resolve a ref to its manifest
+const manifest = await space.resolveRef('my-dataset')
+console.log(manifest.tracks)
+```
+
+## Documentation
+
+- [Installation](https://dreamdb.dreamlake.ai/installation)
+- [TypeScript SDK Reference](https://dreamdb.dreamlake.ai/typescript-sdk)
+- [Protocol Specification](https://dreamdb.dreamlake.ai/spec-overview)
+- [Browser Demo](https://demo.dreamdb.dreamlake.ai)
+
+## License
+
+MIT

package/dist/index.cjs

@@ -27,6 +27,7 @@ __export(index_exports, {
   SCALAR_ENCODING_MAP: () => SCALAR_ENCODING_MAP,
   Schema: () => Schema,
   Space: () => Space,
+  ZERO_SPATIAL_KEY: () => ZERO_SPATIAL_KEY,
   base32ToBytes: () => base32ToBytes,
   bitsFromProjections: () => bitsFromProjections,
   bitsToBase2: () => bitsToBase2,
@@ -376,16 +377,33 @@ var S3Backend = class {
     return path;
   }
   async list(prefix) {
-    const url = `${this.baseUrl}?list-type=2&prefix=${encodeURIComponent(prefix)}`;
-    const r = await fetch(url);
-    if (!r.ok) throw new Error(`LIST ${url}: HTTP ${r.status}`);
-    const text = await r.text();
     const keys = [];
-    const regex = /<Key>([^<]+)<\/Key>/g;
-    let match;
-    while ((match = regex.exec(text)) !== null) {
-      keys.push(match[1]);
-    }
+    let continuationToken;
+    do {
+      let url = `${this.baseUrl}?list-type=2&prefix=${encodeURIComponent(prefix)}`;
+      if (continuationToken) {
+        url += `&continuation-token=${encodeURIComponent(continuationToken)}`;
+      }
+      const r = await fetch(url);
+      if (!r.ok) throw new Error(`LIST ${url}: HTTP ${r.status}`);
+      const text = await r.text();
+      const regex = /<Key>([^<]+)<\/Key>/g;
+      let match;
+      while ((match = regex.exec(text)) !== null) {
+        keys.push(match[1]);
+      }
+      const truncatedMatch = text.match(/<IsTruncated>(\w+)<\/IsTruncated>/);
+      const isTruncated = truncatedMatch?.[1] === "true";
+      if (isTruncated) {
+        const tokenMatch = text.match(
+          /<NextContinuationToken>([^<]+)<\/NextContinuationToken>/
+        );
+        continuationToken = tokenMatch?.[1];
+        if (!continuationToken) break;
+      } else {
+        continuationToken = void 0;
+      }
+    } while (continuationToken);
     return keys;
   }
   async delete(path) {
@@ -1365,6 +1383,17 @@ var Space = class _Space {
   }
 };
 
+// src/types.ts
+var ZERO_SPATIAL_KEY = "0000000000000000";
+var SCALAR_ENCODING_MAP = {
+  categorical: "cat",
+  bool: "bool",
+  int: "i64",
+  float: "f64",
+  string: "str",
+  timestamp: "ts"
+};
+
 // src/dataset.ts
 var Schema = class {
   fields = [];
@@ -1674,7 +1703,7 @@ var Dataset = class _Dataset {
           const fragHash = contentHash(data);
           const fragHashB32 = bytesToBase32(fragHash);
           await this.backend.put(
-            `${this._timelineHash}/${modality}/0000000000000000/${fragHashB32}`,
+            `${this._timelineHash}/${modality}/${ZERO_SPATIAL_KEY}/${fragHashB32}`,
             data
           );
           newEntries.push([
@@ -1922,7 +1951,7 @@ var Dataset = class _Dataset {
         const fragHash = bytesToBase32(e[3]);
         try {
           const fragBytes = await this.backend.get(
-            `${this._timelineHash}/${bfMod}/0000000000000000/${fragHash}`
+            `${this._timelineHash}/${bfMod}/${ZERO_SPATIAL_KEY}/${fragHash}`
           );
           fieldData.set(tStart, fragBytes);
         } catch {
@@ -2225,16 +2254,6 @@ function schemaFromCbor(obj) {
   }
   return schema;
 }
-
-// src/types.ts
-var SCALAR_ENCODING_MAP = {
-  categorical: "cat",
-  bool: "bool",
-  int: "i64",
-  float: "f64",
-  string: "str",
-  timestamp: "ts"
-};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   Dataset,
@@ -2244,6 +2263,7 @@ var SCALAR_ENCODING_MAP = {
   SCALAR_ENCODING_MAP,
   Schema,
   Space,
+  ZERO_SPATIAL_KEY,
   base32ToBytes,
   bitsFromProjections,
   bitsToBase2,