@tungthedev/streams-server 0.2.0

package/src/runtime_memory_sampler.ts

@@ -0,0 +1,237 @@
+import { mkdirSync, createWriteStream, type WriteStream } from "node:fs";
+import { dirname, extname } from "node:path";
+import { isMainThread, threadId } from "node:worker_threads";
+import { readLinuxStatusRssBreakdown } from "./runtime_memory";
+
+type BunJscModule = {
+  heapStats?: () => unknown;
+  memoryUsage?: () => unknown;
+};
+
+type PhaseMeta = Record<string, unknown>;
+
+type ActivePhase = {
+  key: number;
+  label: string;
+  startedAtMs: number;
+  meta: PhaseMeta;
+};
+
+function isPromiseLike<T>(value: unknown): value is PromiseLike<T> {
+  return typeof value === "object" && value != null && "then" in value && typeof (value as { then?: unknown }).then === "function";
+}
+
+function insertScopeSuffix(path: string, scope: string): string {
+  if (scope === "main") return path;
+  const extension = extname(path);
+  if (extension === "") return `${path}.${scope}`;
+  return `${path.slice(0, -extension.length)}.${scope}${extension}`;
+}
+
+export class RuntimeMemorySampler {
+  private readonly outputPath: string;
+  private readonly intervalMs: number;
+  private readonly scope: string;
+  private stream: WriteStream | null = null;
+  private timer: any | null = null;
+  private activePhases = new Map<number, ActivePhase>();
+  private nextPhaseKey = 0;
+  private jscModule: BunJscModule | null | undefined;
+  private sampling = false;
+  private subsystemProvider: (() => unknown) | null = null;
+
+  constructor(path: string, opts: { intervalMs?: number; scope?: string } = {}) {
+    this.scope = opts.scope?.trim() || "main";
+    this.outputPath = insertScopeSuffix(path, this.scope);
+    this.intervalMs = Math.max(100, opts.intervalMs ?? 1_000);
+  }
+
+  start(): void {
+    if (this.stream) return;
+    mkdirSync(dirname(this.outputPath), { recursive: true });
+    this.stream = createWriteStream(this.outputPath, { flags: "a" });
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "sampler_start",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      is_main_thread: isMainThread,
+    });
+    void this.sample("startup");
+    this.timer = setInterval(() => {
+      void this.sample("interval");
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    void this.sample("shutdown");
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "sampler_stop",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      is_main_thread: isMainThread,
+    });
+    this.stream?.end();
+    this.stream = null;
+  }
+
+  enter(label: string, meta: PhaseMeta = {}): () => void {
+    if (!this.stream) return () => {};
+    const key = ++this.nextPhaseKey;
+    const phase: ActivePhase = {
+      key,
+      label,
+      startedAtMs: Date.now(),
+      meta,
+    };
+    this.activePhases.set(key, phase);
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "phase_enter",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      label,
+      meta,
+      active_phase_count: this.activePhases.size,
+    });
+    void this.sample("phase_enter", { label, meta });
+    return () => {
+      const current = this.activePhases.get(key);
+      if (!current) return;
+      this.activePhases.delete(key);
+      this.writeLine({
+        ts: new Date().toISOString(),
+        kind: "phase_exit",
+        scope: this.scope,
+        pid: process.pid,
+        thread_id: threadId,
+        label: current.label,
+        meta: current.meta,
+        duration_ms: Date.now() - current.startedAtMs,
+        active_phase_count: this.activePhases.size,
+      });
+      void this.sample("phase_exit", { label: current.label, meta: current.meta });
+    };
+  }
+
+  capture(reason: string, data: PhaseMeta = {}): void {
+    if (!this.stream) return;
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "marker",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      reason,
+      data,
+    });
+    void this.sample(reason, data);
+  }
+
+  setSubsystemProvider(provider: (() => unknown) | null): void {
+    this.subsystemProvider = provider;
+  }
+
+  track<T>(label: string, meta: PhaseMeta, fn: () => T): T {
+    const leave = this.enter(label, meta);
+    try {
+      const result = fn();
+      if (isPromiseLike(result)) {
+        return (Promise.resolve(result).finally(() => leave()) as unknown) as T;
+      }
+      leave();
+      return result;
+    } catch (error) {
+      leave();
+      throw error;
+    }
+  }
+
+  private async sample(reason: string, data: PhaseMeta = {}): Promise<void> {
+    if (!this.stream || this.sampling) return;
+    this.sampling = true;
+    try {
+      const activePhases = Array.from(this.activePhases.values()).map((phase) => ({
+        key: phase.key,
+        label: phase.label,
+        started_at: new Date(phase.startedAtMs).toISOString(),
+        duration_ms: Date.now() - phase.startedAtMs,
+        meta: phase.meta,
+      }));
+      const primary = activePhases.length > 0 ? activePhases[activePhases.length - 1]! : null;
+      this.writeLine({
+        ts: new Date().toISOString(),
+        kind: "sample",
+        scope: this.scope,
+        pid: process.pid,
+        thread_id: threadId,
+        is_main_thread: isMainThread,
+        reason,
+        data,
+        process_memory_usage: process.memoryUsage(),
+        linux_status_rss: readLinuxStatusRssBreakdown(0),
+        jsc_heap_stats: await this.readJscHeapStats(),
+        jsc_memory_usage: await this.readJscMemoryUsage(),
+        memory_subsystems: this.readSubsystems(),
+        active_phases: activePhases,
+        primary_phase: primary ? { label: primary.label, duration_ms: primary.duration_ms, meta: primary.meta } : null,
+      });
+    } finally {
+      this.sampling = false;
+    }
+  }
+
+  private async readJscHeapStats(): Promise<unknown> {
+    const jsc = await this.loadJscModule();
+    if (!jsc || typeof jsc.heapStats !== "function") return null;
+    try {
+      return jsc.heapStats();
+    } catch {
+      return null;
+    }
+  }
+
+  private async readJscMemoryUsage(): Promise<unknown> {
+    const jsc = await this.loadJscModule();
+    if (!jsc || typeof jsc.memoryUsage !== "function") return null;
+    try {
+      return jsc.memoryUsage();
+    } catch {
+      return null;
+    }
+  }
+
+  private async loadJscModule(): Promise<BunJscModule | null> {
+    if (this.jscModule !== undefined) return this.jscModule;
+    if (typeof (globalThis as { Bun?: unknown }).Bun === "undefined") {
+      this.jscModule = null;
+      return this.jscModule;
+    }
+    try {
+      this.jscModule = (await import("bun:jsc")) as BunJscModule;
+    } catch {
+      this.jscModule = null;
+    }
+    return this.jscModule;
+  }
+
+  private writeLine(payload: Record<string, unknown>): void {
+    if (!this.stream) return;
+    this.stream.write(`${JSON.stringify(payload)}\n`);
+  }
+
+  private readSubsystems(): unknown {
+    if (!this.subsystemProvider) return null;
+    try {
+      return this.subsystemProvider();
+    } catch {
+      return null;
+    }
+  }
+}

package/src/schema/lens_schema.ts

@@ -0,0 +1,290 @@
+export const DURABLE_LENS_V1_SCHEMA: any = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "https://example.com/schemas/durable-lens-v1.schema.json",
+  title: "Durable Stream Lens Spec",
+  type: "object",
+  additionalProperties: false,
+  required: ["apiVersion", "schema", "from", "to", "ops"],
+  properties: {
+    apiVersion: {
+      type: "string",
+      const: "durable.lens/v1",
+    },
+    schema: {
+      type: "string",
+      minLength: 1,
+      description: "Logical stream schema/type name (e.g., 'Task').",
+    },
+    from: {
+      type: "integer",
+      minimum: 0,
+      description: "Source schema version.",
+    },
+    to: {
+      type: "integer",
+      minimum: 0,
+      description: "Target schema version.",
+    },
+    description: {
+      type: "string",
+    },
+    ops: {
+      type: "array",
+      minItems: 1,
+      items: { $ref: "#/$defs/op" },
+    },
+  },
+  $defs: {
+    jsonPointer: {
+      type: "string",
+      description: "RFC 6901 JSON Pointer. Empty string refers to the document root.",
+      pattern: "^(?:/(?:[^~/]|~0|~1)*)*$",
+    },
+    jsonScalar: {
+      description: "JSON scalar value.",
+      type: ["string", "number", "integer", "boolean", "null"],
+    },
+    embeddedJsonSchema: {
+      description: "An embedded JSON Schema fragment (object or boolean).",
+      anyOf: [{ type: "object" }, { type: "boolean" }],
+    },
+    mapTransform: {
+      type: "object",
+      additionalProperties: false,
+      required: ["map"],
+      properties: {
+        map: {
+          type: "object",
+          description: "Mapping table for values. Keys must be strings; values are JSON scalars.",
+          additionalProperties: { $ref: "#/$defs/jsonScalar" },
+        },
+        default: {
+          $ref: "#/$defs/jsonScalar",
+          description: "Default output value used when input is not found in map.",
+        },
+      },
+    },
+    builtinTransform: {
+      type: "object",
+      additionalProperties: false,
+      required: ["builtin"],
+      properties: {
+        builtin: {
+          type: "string",
+          minLength: 1,
+          description: "Name of a built-in, version-stable transform implemented by the system.",
+        },
+      },
+    },
+    convertTransform: {
+      description: "Restricted conversion: either a total mapping table (+ optional default) or a built-in transform.",
+      oneOf: [{ $ref: "#/$defs/mapTransform" }, { $ref: "#/$defs/builtinTransform" }],
+    },
+    opRename: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "from", "to"],
+      properties: {
+        op: { const: "rename" },
+        from: { $ref: "#/$defs/jsonPointer" },
+        to: { $ref: "#/$defs/jsonPointer" },
+      },
+    },
+    opCopy: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "from", "to"],
+      properties: {
+        op: { const: "copy" },
+        from: { $ref: "#/$defs/jsonPointer" },
+        to: { $ref: "#/$defs/jsonPointer" },
+      },
+    },
+    opAdd: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "path", "schema"],
+      properties: {
+        op: { const: "add" },
+        path: { $ref: "#/$defs/jsonPointer" },
+        schema: { $ref: "#/$defs/embeddedJsonSchema" },
+        default: {
+          description: "Default value to insert if the field is missing. If omitted, the runtime may derive a default when possible.",
+          type: ["object", "array", "string", "number", "integer", "boolean", "null"],
+        },
+      },
+    },
+    opRemove: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "path", "schema"],
+      properties: {
+        op: { const: "remove" },
+        path: { $ref: "#/$defs/jsonPointer" },
+        schema: {
+          $ref: "#/$defs/embeddedJsonSchema",
+          description: "Schema of the removed field (required so the transformation remains declarative and reversible/validatable).",
+        },
+        default: {
+          description: "Default to use when reconstructing the field (e.g., during backward reasoning/validation).",
+          type: ["object", "array", "string", "number", "integer", "boolean", "null"],
+        },
+      },
+    },
+    opHoist: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "host", "name", "to"],
+      properties: {
+        op: { const: "hoist" },
+        host: {
+          $ref: "#/$defs/jsonPointer",
+          description: "Pointer to an object field that contains the nested value.",
+        },
+        name: {
+          type: "string",
+          minLength: 1,
+          description: "Field name inside host to move outward.",
+        },
+        to: {
+          $ref: "#/$defs/jsonPointer",
+          description: "Destination pointer for the hoisted value.",
+        },
+        removeFromHost: {
+          type: "boolean",
+          default: true,
+          description: "If true, remove the nested field from the host after hoisting.",
+        },
+      },
+    },
+    opPlunge: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "from", "host", "name"],
+      properties: {
+        op: { const: "plunge" },
+        from: {
+          $ref: "#/$defs/jsonPointer",
+          description: "Pointer to the source field to move inward.",
+        },
+        host: {
+          $ref: "#/$defs/jsonPointer",
+          description: "Pointer to the destination object field.",
+        },
+        name: {
+          type: "string",
+          minLength: 1,
+          description: "Field name inside host to receive the value.",
+        },
+        createHost: {
+          type: "boolean",
+          default: true,
+          description: "If true, create the destination host object if missing.",
+        },
+        removeFromSource: {
+          type: "boolean",
+          default: true,
+          description: "If true, remove the source field after plunging.",
+        },
+      },
+    },
+    opWrap: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "path", "mode"],
+      properties: {
+        op: { const: "wrap" },
+        path: { $ref: "#/$defs/jsonPointer" },
+        mode: {
+          type: "string",
+          enum: ["singleton"],
+          description: "singleton: x -> [x]",
+        },
+        reverseMode: {
+          type: "string",
+          enum: ["first"],
+          default: "first",
+          description: "When reversing array->scalar, choose 'first'.",
+        },
+      },
+    },
+    opHead: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "path"],
+      properties: {
+        op: { const: "head" },
+        path: { $ref: "#/$defs/jsonPointer" },
+        reverseMode: {
+          type: "string",
+          enum: ["singleton"],
+          default: "singleton",
+          description: "When reversing scalar->array, wrap as [scalar].",
+        },
+      },
+    },
+    opConvert: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "path", "fromType", "toType", "forward", "backward"],
+      properties: {
+        op: { const: "convert" },
+        path: { $ref: "#/$defs/jsonPointer" },
+        fromType: {
+          type: "string",
+          enum: ["string", "number", "integer", "boolean", "null", "object", "array"],
+        },
+        toType: {
+          type: "string",
+          enum: ["string", "number", "integer", "boolean", "null", "object", "array"],
+        },
+        forward: { $ref: "#/$defs/convertTransform" },
+        backward: { $ref: "#/$defs/convertTransform" },
+      },
+    },
+    opIn: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "path", "ops"],
+      properties: {
+        op: { const: "in" },
+        path: { $ref: "#/$defs/jsonPointer" },
+        ops: {
+          type: "array",
+          minItems: 1,
+          items: { $ref: "#/$defs/op" },
+        },
+      },
+    },
+    opMap: {
+      type: "object",
+      additionalProperties: false,
+      required: ["op", "path", "ops"],
+      properties: {
+        op: { const: "map" },
+        path: { $ref: "#/$defs/jsonPointer" },
+        ops: {
+          type: "array",
+          minItems: 1,
+          items: { $ref: "#/$defs/op" },
+        },
+      },
+    },
+    op: {
+      description: "A single lens operation.",
+      oneOf: [
+        { $ref: "#/$defs/opRename" },
+        { $ref: "#/$defs/opCopy" },
+        { $ref: "#/$defs/opAdd" },
+        { $ref: "#/$defs/opRemove" },
+        { $ref: "#/$defs/opHoist" },
+        { $ref: "#/$defs/opPlunge" },
+        { $ref: "#/$defs/opWrap" },
+        { $ref: "#/$defs/opHead" },
+        { $ref: "#/$defs/opConvert" },
+        { $ref: "#/$defs/opIn" },
+        { $ref: "#/$defs/opMap" },
+      ],
+    },
+  },
+};