@prisma/streams-server 0.1.1 → 0.1.3

package/src/uploader.ts

@@ -13,15 +13,26 @@ import { LruCache } from "./util/lru";
 import type { StatsCollector } from "./stats";
 import type { BackpressureGate } from "./backpressure";
 import { dsError } from "./util/ds_error.ts";
+import { RuntimeMemorySampler } from "./runtime_memory_sampler";
 
 export type UploaderController = {
   start(): void;
   stop(hard?: boolean): void;
   countSegmentsWaiting(): number;
-  setHooks(hooks: { onSegmentsUploaded?: (stream: string) => void } | undefined): void;
+  getMemoryStats?: () => {
+    inflight_segments: number;
+    inflight_segment_bytes: number;
+    manifest_inflight_streams: number;
+  };
+  setHooks(hooks: UploaderHooks | undefined): void;
   publishManifest(stream: string): Promise<void>;
 };
 
+export type UploaderHooks = {
+  onSegmentsUploaded?: (stream: string) => void;
+  onMetadataChanged?: (stream: string) => void;
+};
+
 export class Uploader {
   private readonly config: Config;
   private readonly db: SqliteDurableStore;
@@ -29,13 +40,15 @@ export class Uploader {
   private readonly diskCache?: SegmentDiskCache;
   private readonly stats?: StatsCollector;
   private readonly gate?: BackpressureGate;
+  private readonly memorySampler?: RuntimeMemorySampler;
   private timer: any | null = null;
   private running = false;
   private stopping = false;
   private readonly inflight = new Set<string>();
   private readonly failures = new FailureTracker(1024);
-  private hooks?: { onSegmentsUploaded?: (stream: string) => void };
+  private hooks?: UploaderHooks;
   private readonly manifestInflight = new Set<string>();
+  private inflightSegmentBytes = 0;
 
   constructor(
     config: Config,
@@ -44,7 +57,8 @@ export class Uploader {
     diskCache?: SegmentDiskCache,
     stats?: StatsCollector,
     gate?: BackpressureGate,
-    hooks?: { onSegmentsUploaded?: (stream: string) => void }
+    hooks?: UploaderHooks,
+    memorySampler?: RuntimeMemorySampler
   ) {
     this.config = config;
     this.db = db;
@@ -53,9 +67,10 @@ export class Uploader {
     this.stats = stats;
     this.gate = gate;
     this.hooks = hooks;
+    this.memorySampler = memorySampler;
   }
 
-  setHooks(hooks: { onSegmentsUploaded?: (stream: string) => void } | undefined): void {
+  setHooks(hooks: UploaderHooks | undefined): void {
     this.hooks = hooks;
   }
 
@@ -78,12 +93,20 @@ export class Uploader {
     return this.db.countPendingSegments();
   }
 
+  getMemoryStats(): { inflight_segments: number; inflight_segment_bytes: number; manifest_inflight_streams: number } {
+    return {
+      inflight_segments: this.inflight.size,
+      inflight_segment_bytes: this.inflightSegmentBytes,
+      manifest_inflight_streams: this.manifestInflight.size,
+    };
+  }
+
   private async tick(): Promise<void> {
     if (this.stopping) return;
     if (this.running) return;
     this.running = true;
     try {
-      const pending = this.db.pendingUploadSegments(1000);
+      const pending = this.db.pendingUploadHeads(1000);
       if (pending.length === 0) return;
 
       // Upload with bounded concurrency.
@@ -141,6 +164,7 @@ export class Uploader {
       if (!seg) return;
       if (this.inflight.has(seg.segment_id)) continue;
       this.inflight.add(seg.segment_id);
+      this.inflightSegmentBytes += Math.max(0, seg.size_bytes);
       try {
         try {
           await this.uploadOne(seg);
@@ -155,6 +179,7 @@ export class Uploader {
         }
       } finally {
         this.inflight.delete(seg.segment_id);
+        this.inflightSegmentBytes = Math.max(0, this.inflightSegmentBytes - Math.max(0, seg.size_bytes));
       }
     }
   }
@@ -163,6 +188,11 @@ export class Uploader {
     if (this.stopping) return;
     const shash = streamHash16Hex(seg.stream);
     const objectKey = segmentObjectKey(shash, seg.segment_index);
+    const leaveUploadPhase = this.memorySampler?.enter("upload", {
+      stream: seg.stream,
+      segment_index: seg.segment_index,
+      size_bytes: seg.size_bytes,
+    });
     try {
       const res = await retry(
         async () => {
@@ -180,11 +210,14 @@ export class Uploader {
         }
       );
       this.db.markSegmentUploaded(seg.segment_id, res.etag, this.db.nowMs());
+      this.hooks?.onMetadataChanged?.(seg.stream);
       if (this.stats) this.stats.recordUploadedBytes(seg.size_bytes);
       if (this.gate) this.gate.adjustOnUpload(seg.size_bytes);
     } catch (e) {
       this.failures.recordFailure(seg.stream);
       throw e;
+    } finally {
+      leaveUploadPhase?.();
     }
   }
 
@@ -218,6 +251,9 @@ export class Uploader {
 
       const uploadedThrough =
         uploadedPrefix === 0 ? -1n : readU64LE(meta.segment_offsets, (uploadedPrefix - 1) * 8) - 1n;
+      const unpublishedWalBytes = this.db.getWalBytesAfterOffset(stream, uploadedThrough);
+      const publishedLogicalSizeBytes =
+        srow.logical_size_bytes > unpublishedWalBytes ? srow.logical_size_bytes - unpublishedWalBytes : 0n;
 
       const manifestRow = this.db.getManifestRow(stream);
       const generation = manifestRow.generation + 1;
@@ -225,15 +261,49 @@ export class Uploader {
       const indexState = this.db.getIndexState(stream);
       const indexRuns = this.db.listIndexRuns(stream);
       const retiredRuns = this.db.listRetiredIndexRuns(stream);
+      const secondaryIndexStates = this.db.listSecondaryIndexStates(stream);
+      const secondaryIndexRuns = secondaryIndexStates.flatMap((state) => this.db.listSecondaryIndexRuns(stream, state.index_name));
+      const retiredSecondaryIndexRuns = secondaryIndexStates.flatMap((state) =>
+        this.db.listRetiredSecondaryIndexRuns(stream, state.index_name)
+      );
+      const lexiconIndexStates = this.db.listLexiconIndexStates(stream);
+      const lexiconIndexRuns = lexiconIndexStates.flatMap((state) =>
+        this.db.listLexiconIndexRuns(stream, state.source_kind, state.source_name)
+      );
+      const retiredLexiconIndexRuns = lexiconIndexStates.flatMap((state) =>
+        this.db.listRetiredLexiconIndexRuns(stream, state.source_kind, state.source_name)
+      );
+      const searchCompanionPlan = this.db.getSearchCompanionPlan(stream);
+      const searchSegmentCompanions = this.db.listSearchSegmentCompanions(stream);
+      let profileJson: Record<string, any> | null = null;
+      const profileRow = this.db.getStreamProfile(stream);
+      if (profileRow) {
+        try {
+          profileJson = JSON.parse(profileRow.profile_json);
+        } catch {
+          this.failures.recordFailure(stream);
+          throw dsError(`invalid profile_json for ${stream}`);
+        }
+      }
       const manifestRes = buildManifestResult({
         streamName: stream,
         streamRow: srow,
+        publishedLogicalSizeBytes,
+        profileJson,
         segmentMeta: meta,
         uploadedPrefixCount: uploadedPrefix,
         generation,
         indexState,
         indexRuns,
         retiredRuns,
+        secondaryIndexStates,
+        secondaryIndexRuns,
+        retiredSecondaryIndexRuns,
+        lexiconIndexStates,
+        lexiconIndexRuns,
+        retiredLexiconIndexRuns,
+        searchCompanionPlan,
+        searchSegmentCompanions,
       });
       if (Result.isError(manifestRes)) {
         this.failures.recordFailure(stream);
@@ -261,7 +331,8 @@ export class Uploader {
       }
 
       // Commit point: advance uploaded_through and delete WAL prefix.
-      this.db.commitManifest(stream, generation, putRes.etag, this.db.nowMs(), uploadedThrough);
+      this.db.commitManifest(stream, generation, putRes.etag, this.db.nowMs(), uploadedThrough, body.byteLength);
+      this.hooks?.onMetadataChanged?.(stream);
 
       // Local disk cleanup: delete newly uploaded segment files.
       if (uploadedPrefix > prevPrefix) {

package/src/util/bloom256.ts

@@ -1,13 +1,13 @@
 import { dsError } from "./ds_error.ts";
 /**
- * A tiny 256-bit bloom filter (2048 bits) with 3 hash probes.
+ * A tiny 256-bit bloom filter with 3 hash probes.
  *
  * This is used per DSB3 block to cheaply skip blocks during key-filtered reads.
  *
  * Correctness rule: bloom filter may have false positives but MUST NOT have false negatives.
  */
 
-const BITS = 2048n;
+const BITS = 256n;
 const MASK64 = (1n << 64n) - 1n;
 
 function fnv1a64(data: Uint8Array): bigint {

package/src/util/byte_lru.ts

@@ -0,0 +1,73 @@
+export class ByteLruCache<K, V> {
+  private readonly map = new Map<K, { value: V; weight: number }>();
+  private readonly capacityBytes: number;
+  private readonly weigh: (value: V) => number;
+  private usedBytes = 0;
+
+  constructor(maxBytes: number, weigh: (value: V) => number) {
+    this.capacityBytes = Math.max(0, Math.floor(maxBytes));
+    this.weigh = weigh;
+  }
+
+  get maxBytes(): number {
+    return this.capacityBytes;
+  }
+
+  get size(): number {
+    return this.map.size;
+  }
+
+  get totalBytes(): number {
+    return this.usedBytes;
+  }
+
+  get(key: K): V | undefined {
+    const entry = this.map.get(key);
+    if (!entry) return undefined;
+    this.map.delete(key);
+    this.map.set(key, entry);
+    return entry.value;
+  }
+
+  set(key: K, value: V): boolean {
+    const weight = Math.max(0, Math.floor(this.weigh(value)));
+    if (this.capacityBytes <= 0 || weight > this.capacityBytes) {
+      this.delete(key);
+      return false;
+    }
+    const existing = this.map.get(key);
+    if (existing) {
+      this.usedBytes -= existing.weight;
+      this.map.delete(key);
+    }
+    this.map.set(key, { value, weight });
+    this.usedBytes += weight;
+    this.evict();
+    return true;
+  }
+
+  has(key: K): boolean {
+    return this.map.has(key);
+  }
+
+  delete(key: K): boolean {
+    const existing = this.map.get(key);
+    if (!existing) return false;
+    this.usedBytes -= existing.weight;
+    this.map.delete(key);
+    return true;
+  }
+
+  clear(): void {
+    this.map.clear();
+    this.usedBytes = 0;
+  }
+
+  private evict(): void {
+    while (this.usedBytes > this.capacityBytes && this.map.size > 0) {
+      const oldest = this.map.keys().next();
+      if (oldest.done) break;
+      this.delete(oldest.value);
+    }
+  }
+}

package/src/util/lru.ts

@@ -42,4 +42,12 @@ export class LruCache<K, V> {
   clear(): void {
     this.map.clear();
   }
+
+  values(): IterableIterator<V> {
+    return this.map.values();
+  }
+
+  entries(): IterableIterator<[K, V]> {
+    return this.map.entries();
+  }
 }

package/src/util/stream_paths.ts

@@ -26,6 +26,25 @@ export function indexRunObjectKey(streamHash: string, runId: string): string {
   return `streams/${streamHash}/index/${runId}.idx`;
 }
 
+export function secondaryIndexRunObjectKey(streamHash: string, indexName: string, runId: string): string {
+  return `streams/${streamHash}/secondary-index/${encodeURIComponent(indexName)}/${runId}.idx`;
+}
+
+export function lexiconRunObjectKey(
+  streamHash: string,
+  sourceKind: string,
+  sourceName: string,
+  runId: string
+): string {
+  const encodedKind = encodeURIComponent(sourceKind);
+  const encodedName = encodeURIComponent(sourceName === "" ? "__default__" : sourceName);
+  return `streams/${streamHash}/lexicon/${encodedKind}/${encodedName}/${runId}.lex`;
+}
+
+export function searchCompanionObjectKey(streamHash: string, segmentIndex: number, objectId: string): string {
+  return `streams/${streamHash}/segments/${pad16(segmentIndex)}-${objectId}.cix`;
+}
+
 export function localSegmentPath(rootDir: string, streamHash: string, segmentIndex: number): string {
   return `${rootDir}/local/streams/${streamHash}/segments/${pad16(segmentIndex)}.bin`;
 }