@tungthedev/streams-server 0.2.0

package/src/app_local.ts

@@ -0,0 +1,206 @@
+import type { Config } from "./config";
+import { createAppCore, type App } from "./app_core";
+import type { ObjectStore } from "./objectstore/interface";
+import { NullObjectStore } from "./objectstore/null";
+import { StreamReader } from "./reader";
+import type { StreamIndexLookup } from "./index/indexer";
+import type { RoutingKeyLexiconListResult } from "./index/lexicon_indexer";
+import type { StatsCollector } from "./stats";
+import { readSqliteRuntimeMemoryStats } from "./sqlite/runtime_stats";
+import { Result } from "better-result";
+import { SqliteDurableStore } from "./db/db";
+import type { WalControlPlaneStore } from "./store/capabilities";
+
+const TEXT_DECODER = new TextDecoder();
+
+class LocalIndexLookup implements StreamIndexLookup {
+  constructor(private readonly db: SqliteDurableStore) {}
+
+  start(): void {}
+
+  async stop(): Promise<void> {}
+
+  enqueue(_stream: string): void {}
+
+  async candidateSegmentsForRoutingKey(_stream: string, _keyBytes: Uint8Array): Promise<null> {
+    return null;
+  }
+
+  async candidateSegmentsForSecondaryIndex(_stream: string, _indexName: string, _keyBytes: Uint8Array): Promise<null> {
+    return null;
+  }
+
+  async getAggSegmentCompanion(_stream: string, _segmentIndex: number): Promise<null> {
+    return null;
+  }
+
+  async getColSegmentCompanion(_stream: string, _segmentIndex: number): Promise<null> {
+    return null;
+  }
+
+  async getExactSegmentCompanion(_stream: string, _segmentIndex: number): Promise<null> {
+    return null;
+  }
+
+  async getFtsSegmentCompanion(_stream: string, _segmentIndex: number): Promise<null> {
+    return null;
+  }
+
+  async getMetricsBlockSegmentCompanion(_stream: string, _segmentIndex: number): Promise<null> {
+    return null;
+  }
+
+  async listRoutingKeysResult(
+    stream: string,
+    after: string | null,
+    limit: number
+  ): Promise<Result<RoutingKeyLexiconListResult, { kind: string; message: string }>> {
+    const srow = this.db.getStream(stream);
+    if (!srow || this.db.isDeleted(srow)) {
+      return Result.err({ kind: "invalid_lexicon_index", message: "stream not found" });
+    }
+    const safeLimit = Math.max(1, Math.min(limit, 500));
+    const keys = new Set<string>();
+    let scannedWalRows = 0;
+    for (const rec of this.db.iterWalRange(stream, 0n, srow.next_offset - 1n)) {
+      scannedWalRows += 1;
+      const rawKey = rec.routing_key == null ? null : rec.routing_key instanceof Uint8Array ? rec.routing_key : new Uint8Array(rec.routing_key);
+      if (!rawKey || rawKey.byteLength === 0) continue;
+      keys.add(TEXT_DECODER.decode(rawKey));
+    }
+    const sorted = Array.from(keys).sort();
+    const filtered = after == null ? sorted : sorted.filter((key) => key > after);
+    const page = filtered.slice(0, safeLimit);
+    const nextAfter = filtered.length > safeLimit ? page[page.length - 1] ?? null : null;
+    return Result.ok({
+      keys: page,
+      nextAfter,
+      tookMs: 0,
+      coverage: {
+        complete: true,
+        indexedSegments: 0,
+        scannedUploadedSegments: 0,
+        scannedLocalSegments: 0,
+        scannedWalRows,
+        possibleMissingUploadedSegments: 0,
+        possibleMissingLocalSegments: 0,
+      },
+        timing: {
+          lexiconRunGetMs: 0,
+          lexiconDecodeMs: 0,
+          lexiconEnumerateMs: 0,
+          lexiconMergeMs: 0,
+          fallbackScanMs: 0,
+        fallbackSegmentGetMs: 0,
+        fallbackWalScanMs: 0,
+        lexiconRunsLoaded: 0,
+      },
+    });
+  }
+
+  getLocalStorageUsage(_stream: string) {
+    return {
+      routing_index_cache_bytes: 0,
+      exact_index_cache_bytes: 0,
+      companion_cache_bytes: 0,
+      lexicon_index_cache_bytes: 0,
+    };
+  }
+}
+
+export type CreateLocalAppOptions = {
+  stats?: StatsCollector;
+};
+
+function localControlStore(db: SqliteDurableStore): WalControlPlaneStore {
+  return new Proxy(db, {
+    get(target, prop, receiver) {
+      if (prop === "capabilities") {
+        return {
+          ...target.capabilities,
+          manifests: false,
+          schemaPublication: false,
+          internalMetrics: false,
+        };
+      }
+      const value = Reflect.get(target, prop, receiver);
+      return typeof value === "function" ? value.bind(target) : value;
+    },
+  }) as WalControlPlaneStore;
+}
+
+export function createLocalApp(cfg: Config, os?: ObjectStore, opts: CreateLocalAppOptions = {}): App<SqliteDurableStore> {
+  const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.sqliteCacheBytes });
+  const controlStore = localControlStore(db);
+  return createAppCore(cfg, {
+    debugStore: db,
+    touchStore: db.touch,
+    storageStatsStore: db,
+    objectStoreAccountingStore: db,
+    detailsStore: db,
+    lifecycleHooks: {
+      getInitialBackpressureBytes: () => db.sumPendingBytes() + db.sumPendingSegmentBytes(),
+    },
+    store: controlStore,
+    stats: opts.stats,
+    createRuntime: ({ config, registry, memorySampler, memory }) => {
+      const store = os ?? new NullObjectStore();
+      const indexer = new LocalIndexLookup(db);
+      const reader = new StreamReader(
+        config,
+        controlStore,
+        registry,
+        { segmentReads: db, objectStore: store, index: indexer },
+        memorySampler,
+        memory
+      );
+
+      return {
+        reader,
+        indexer,
+        getRuntimeMemorySnapshot: () => {
+          const sqliteRuntime = readSqliteRuntimeMemoryStats();
+          return {
+            subsystems: {
+              heap_estimates: {
+                ingest_queue_payload_bytes: 0,
+              },
+              mapped_files: {},
+              disk_caches: {},
+              configured_budgets: {
+                sqlite_cache_budget_bytes: config.sqliteCacheBytes,
+                worker_sqlite_cache_budget_bytes: config.workerSqliteCacheBytes,
+              },
+              pipeline_buffers: {},
+              sqlite_runtime: {
+                sqlite_memory_used_bytes: sqliteRuntime.memory_used_bytes,
+                sqlite_memory_highwater_bytes: sqliteRuntime.memory_highwater_bytes,
+                sqlite_pagecache_overflow_bytes: sqliteRuntime.pagecache_overflow_bytes,
+                sqlite_pagecache_overflow_highwater_bytes: sqliteRuntime.pagecache_overflow_highwater_bytes,
+              },
+              counts: {
+                ingest_queue_requests: 0,
+                pending_upload_segments: 0,
+                sqlite_pagecache_used_slots: sqliteRuntime.pagecache_used_slots,
+                sqlite_pagecache_used_slots_highwater: sqliteRuntime.pagecache_used_slots_highwater,
+                sqlite_malloc_count: sqliteRuntime.malloc_count,
+                sqlite_malloc_count_highwater: sqliteRuntime.malloc_count_highwater,
+                sqlite_open_connections: sqliteRuntime.open_connections,
+                sqlite_prepared_statements: sqliteRuntime.prepared_statements,
+              },
+            },
+            totals: {
+              heap_estimate_bytes: 0,
+              mapped_file_bytes: 0,
+              disk_cache_bytes: 0,
+              configured_budget_bytes: config.sqliteCacheBytes + config.workerSqliteCacheBytes,
+              pipeline_buffer_bytes: 0,
+              sqlite_runtime_bytes: sqliteRuntime.memory_used_bytes + sqliteRuntime.pagecache_overflow_bytes,
+            },
+          };
+        },
+        start: (): void => {},
+      };
+    },
+  });
+}

package/src/auth.ts

@@ -0,0 +1,124 @@
+import { timingSafeEqual } from "node:crypto";
+import { Result } from "better-result";
+
+export type AuthConfig =
+  | {
+      mode: "none";
+    }
+	  | {
+	      mode: "api-key";
+	      apiKeyBytes: Buffer;
+	    };
+
+export type AuthConfigError = {
+  message: string;
+};
+
+const API_KEY_MIN_LENGTH = 10;
+
+function hasFlagArg(args: string[], flag: string): boolean {
+  return args.includes(flag);
+}
+
+function valuesForOption(args: string[], option: string): string[] {
+  const values: string[] = [];
+  const prefix = `${option}=`;
+  for (let i = 0; i < args.length; i += 1) {
+    const arg = args[i];
+    if (arg === option) {
+      const value = args[i + 1];
+      if (value != null && !value.startsWith("--")) {
+        values.push(value);
+        i += 1;
+      } else {
+        values.push("");
+      }
+      continue;
+    }
+    if (arg.startsWith(prefix)) {
+      values.push(arg.slice(prefix.length));
+    }
+  }
+  return values;
+}
+
+function authConfigError(message: string): Result<AuthConfig, AuthConfigError> {
+  return Result.err({ message });
+}
+
+export function parseAuthConfigResult(args: string[], env: NodeJS.ProcessEnv = process.env): Result<AuthConfig, AuthConfigError> {
+  const noAuth = hasFlagArg(args, "--no-auth");
+  const authStrategyValues = valuesForOption(args, "--auth-strategy");
+  const hasAuthStrategy = authStrategyValues.length > 0;
+
+  if (noAuth && hasAuthStrategy) {
+    return authConfigError("invalid auth configuration: provide exactly one of --no-auth or --auth-strategy api-key");
+  }
+  if (!noAuth && !hasAuthStrategy) {
+    return authConfigError("missing auth configuration: expected --no-auth or --auth-strategy api-key");
+  }
+  if (noAuth) {
+    return Result.ok({ mode: "none" });
+  }
+  if (authStrategyValues.length !== 1 || authStrategyValues[0] !== "api-key") {
+    return authConfigError("invalid --auth-strategy (expected: api-key)");
+  }
+
+  const apiKey = env.API_KEY;
+  if (apiKey == null || apiKey.length < API_KEY_MIN_LENGTH) {
+    return authConfigError(`API_KEY must be present and contain at least ${API_KEY_MIN_LENGTH} characters`);
+  }
+
+  return Result.ok({
+    mode: "api-key",
+    apiKeyBytes: Buffer.from(apiKey, "utf8"),
+  });
+}
+
+function unauthorized(): Response {
+  return new Response(
+    JSON.stringify({
+      error: {
+        code: "unauthorized",
+        message: "unauthorized",
+      },
+    }),
+    {
+      status: 401,
+      headers: {
+        "content-type": "application/json; charset=utf-8",
+        "cache-control": "no-store",
+        "x-content-type-options": "nosniff",
+        "www-authenticate": "Bearer",
+      },
+    }
+  );
+}
+
+function parseBearerCredential(header: string | null): string | null {
+  if (header == null) return null;
+  const match = /^Bearer (.+)$/i.exec(header);
+  return match?.[1] ?? null;
+}
+
+function credentialsMatch(config: Extract<AuthConfig, { mode: "api-key" }>, credential: string): boolean {
+  const credentialBytes = Buffer.from(credential, "utf8");
+  const paddedCredential = Buffer.alloc(config.apiKeyBytes.length);
+  credentialBytes.copy(paddedCredential, 0, 0, Math.min(credentialBytes.length, paddedCredential.length));
+  const bytesMatch = timingSafeEqual(config.apiKeyBytes, paddedCredential);
+  return credentialBytes.length === config.apiKeyBytes.length && bytesMatch;
+}
+
+export function authenticateRequest(config: AuthConfig, request: Request): Response | null {
+  if (config.mode === "none") return null;
+  const credential = parseBearerCredential(request.headers.get("authorization"));
+  if (credential == null || !credentialsMatch(config, credential)) {
+    return unauthorized();
+  }
+  return null;
+}
+
+export function withAuth(config: AuthConfig, fetch: (request: Request) => Promise<Response>): (request: Request) => Promise<Response> {
+  if (config.mode === "none") return fetch;
+  return async (request: Request): Promise<Response> => authenticateRequest(config, request) ?? fetch(request);
+}

package/src/auto_tune.ts

@@ -0,0 +1,69 @@
+export type AutoTuneConfig = {
+  segmentMaxMiB: number;
+  segmentTargetRows: number;
+  segmentCacheMb: number;
+  indexCheckMs: number;
+  sqliteCacheMb: number;
+  workerSqliteCacheMb: number;
+  indexMemMb: number;
+  lexiconIndexCacheMb: number;
+  searchCompanionTocCacheMb: number;
+  searchCompanionSectionCacheMb: number;
+  ingestBatchMb: number;
+  ingestQueueMb: number;
+  ingestConcurrency: number;
+  readConcurrency: number;
+  searchConcurrency: number;
+  asyncIndexConcurrency: number;
+  indexBuildConcurrency: number;
+  indexCompactConcurrency: number;
+  segmenterWorkers: number;
+  uploadConcurrency: number;
+  searchCompanionBatchSegments: number;
+  searchCompanionYieldBlocks: number;
+};
+
+export const AUTO_TUNE_PRESETS = [256, 512, 1024, 2048, 4096, 8192] as const;
+
+export function memoryLimitForPreset(preset: number): number {
+  return preset === 256 ? 300 : preset;
+}
+
+export function tuneForPreset(p: number): AutoTuneConfig {
+  return {
+    // <=1 GiB hosts need smaller cut units because segment build/compression
+    // can transiently hold several encoded copies of the candidate rows.
+    segmentMaxMiB: p <= 1024 ? 8 : 16,
+    segmentTargetRows: p <= 1024 ? 50_000 : 100_000,
+    // The 1 GiB Compute host only has about 685 MiB of usable RSS after the
+    // platform clamp, so it cannot afford a persistent 256 MiB local segment
+    // cache on top of active ingest and background reads.
+    segmentCacheMb: p >= 2048 ? 256 : 0,
+    // Small hosts defer background sweeps so routing/exact backfill does not
+    // immediately start re-reading uploaded history during a large ingest burst.
+    indexCheckMs: p >= 2048 ? 1_000 : 3_600_000,
+    sqliteCacheMb: Math.max(8, Math.floor(p / 16)),
+    workerSqliteCacheMb: Math.max(8, Math.min(32, Math.floor(p / 128))),
+    indexMemMb: Math.max(4, Math.floor(p / 64)),
+    lexiconIndexCacheMb: p >= 8192 ? 256 : p >= 4096 ? 128 : p >= 2048 ? 64 : p >= 1024 ? 32 : p >= 512 ? 16 : 8,
+    searchCompanionTocCacheMb: p >= 8192 ? 4 : p >= 4096 ? 2 : 1,
+    searchCompanionSectionCacheMb: p >= 8192 ? 128 : p >= 4096 ? 64 : p >= 2048 ? 32 : p >= 1024 ? 16 : 8,
+    // Keep append working sets tighter on <=2 GiB presets because the request path
+    // still holds multiple copies of JSON batches while normalizing and queuing.
+    ingestBatchMb: p >= 8192 ? 64 : p >= 4096 ? 16 : p >= 2048 ? 8 : p >= 1024 ? 4 : 2,
+    ingestQueueMb: p >= 8192 ? 128 : p >= 4096 ? 64 : p >= 2048 ? 32 : p >= 1024 ? 16 : 8,
+    ingestConcurrency: p >= 8192 ? 8 : p >= 4096 ? 4 : p >= 1024 ? 2 : 1,
+    readConcurrency: p >= 8192 ? 16 : p >= 4096 ? 8 : p >= 1024 ? 4 : 2,
+    searchConcurrency: p >= 8192 ? 8 : p >= 4096 ? 4 : p >= 1024 ? 2 : 1,
+    asyncIndexConcurrency: p >= 8192 ? 4 : p >= 4096 ? 2 : 1,
+    // Keep <=2 GiB presets single-lane for background work. These hosts do not
+    // have enough headroom for append, segment cut, upload, and companion work
+    // to overlap aggressively under the GH Archive "all" workload.
+    indexBuildConcurrency: p >= 8192 ? 4 : p >= 4096 ? 2 : 1,
+    indexCompactConcurrency: p >= 8192 ? 4 : p >= 4096 ? 2 : 1,
+    segmenterWorkers: p >= 8192 ? 4 : p >= 4096 ? 2 : p >= 2048 ? 1 : 0,
+    uploadConcurrency: p >= 8192 ? 8 : p >= 4096 ? 4 : p >= 2048 ? 2 : 1,
+    searchCompanionBatchSegments: p >= 8192 ? 4 : p >= 4096 ? 2 : 1,
+    searchCompanionYieldBlocks: p >= 8192 ? 4 : p >= 4096 ? 2 : 1,
+  };
+}

package/src/backpressure.ts

@@ -0,0 +1,66 @@
+export class BackpressureGate {
+  private readonly maxBytes: number;
+  private currentBytes: number;
+  private reservedBytes: number;
+
+  constructor(maxBytes: number, initialBytes: number) {
+    this.maxBytes = maxBytes;
+    this.currentBytes = Math.max(0, initialBytes);
+    this.reservedBytes = 0;
+  }
+
+  enabled(): boolean {
+    return this.maxBytes > 0;
+  }
+
+  reserve(bytes: number): boolean {
+    if (this.maxBytes <= 0) return true;
+    if (bytes <= 0) return true;
+    if (this.currentBytes + this.reservedBytes + bytes > this.maxBytes) return false;
+    this.reservedBytes += bytes;
+    return true;
+  }
+
+  commit(bytes: number, reservedBytes: number = bytes): void {
+    if (this.maxBytes <= 0) return;
+    if (bytes <= 0) return;
+    if (reservedBytes > 0) this.reservedBytes = Math.max(0, this.reservedBytes - reservedBytes);
+    this.currentBytes += bytes;
+  }
+
+  release(bytes: number): void {
+    if (this.maxBytes <= 0) return;
+    if (bytes <= 0) return;
+    this.reservedBytes = Math.max(0, this.reservedBytes - bytes);
+  }
+
+  adjustOnSeal(payloadBytes: number, segmentBytes: number): void {
+    if (this.maxBytes <= 0) return;
+    const delta = segmentBytes - payloadBytes;
+    this.currentBytes = Math.max(0, this.currentBytes + delta);
+  }
+
+  adjustOnUpload(segmentBytes: number): void {
+    if (this.maxBytes <= 0) return;
+    this.currentBytes = Math.max(0, this.currentBytes - segmentBytes);
+  }
+
+  adjustOnWalTrim(payloadBytes: number): void {
+    if (this.maxBytes <= 0) return;
+    if (payloadBytes <= 0) return;
+    this.currentBytes = Math.max(0, this.currentBytes - payloadBytes);
+  }
+
+  getCurrentBytes(): number {
+    return this.currentBytes;
+  }
+
+  getMaxBytes(): number {
+    return this.maxBytes;
+  }
+
+  isOverLimit(): boolean {
+    if (this.maxBytes <= 0) return false;
+    return this.currentBytes + this.reservedBytes >= this.maxBytes;
+  }
+}