effect-start 0.26.0 → 0.28.0

package/src/tower/TowerStore.ts

@@ -0,0 +1,505 @@
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as GlobalValue from "effect/GlobalValue"
+import * as Layer from "effect/Layer"
+import * as MutableRef from "effect/MutableRef"
+import * as PubSub from "effect/PubSub"
+import * as Schema from "effect/Schema"
+import * as Unique from "../Unique.ts"
+import * as SqlClient from "../sql/SqlClient.ts"
+
+export let store: TowerStoreShape = GlobalValue.globalValue(
+  Symbol.for("effect-start/TowerStore"),
+  () => ({
+    prefix: "/tower",
+    sql: undefined as unknown as SqlClient.SqlClient,
+    events: Effect.runSync(PubSub.unbounded<TowerEvent>()),
+    spanCapacity: 1000,
+    logCapacity: 5000,
+    errorCapacity: 1000,
+    metrics: [] as Array<TowerMetricSnapshot>,
+    process: undefined as ProcessStats | undefined,
+  }),
+)
+
+const nextPackedId = (): bigint => Unique.snowflake()
+
+export const nextLogId = () => nextPackedId()
+
+export const nextErrorId = () => nextPackedId()
+
+export const nextSpanId = () => nextPackedId()
+
+export const nextTraceId = () => nextPackedId()
+
+export interface TowerSpan {
+  readonly spanId: bigint
+  readonly traceId: bigint
+  readonly fiberId: string | undefined
+  readonly name: string
+  readonly kind: string
+  readonly parentSpanId: bigint | undefined
+  startTime: bigint
+  endTime: bigint | undefined
+  durationMs: number | undefined
+  status: "started" | "ok" | "error"
+  readonly attributes: Record<string, unknown>
+  readonly events: Array<{ name: string; startTime: bigint; attributes?: Record<string, unknown> }>
+}
+
+export interface TowerLog {
+  readonly id: bigint
+  readonly level: "DEBUG" | "INFO" | "WARNING" | "ERROR" | "FATAL"
+  readonly message: string
+  readonly fiberId: string
+  readonly cause: string | undefined
+  readonly spans: Array<string>
+  readonly annotations: Record<string, unknown>
+}
+
+export interface ProcessStats {
+  readonly pid: number
+  readonly uptime: number
+  readonly memory: {
+    readonly rss: number
+    readonly heapUsed: number
+    readonly heapTotal: number
+    readonly external: number
+    readonly arrayBuffers: number
+  }
+  readonly cpu: { readonly user: number; readonly system: number }
+  readonly resourceUsage: {
+    readonly maxRSS: number
+    readonly minorPageFault: number
+    readonly majorPageFault: number
+    readonly fsRead: number
+    readonly fsWrite: number
+    readonly voluntaryContextSwitches: number
+    readonly involuntaryContextSwitches: number
+  }
+  readonly system: {
+    readonly loadavg: readonly [number, number, number]
+    readonly freemem: number
+    readonly totalmem: number
+    readonly cpuCount: number
+    readonly platform: string
+    readonly arch: string
+  }
+}
+
+export interface TowerErrorDetail {
+  readonly kind: "fail" | "die"
+  readonly tag: string | undefined
+  readonly message: string
+  readonly properties: Record<string, unknown>
+  readonly span: string | undefined
+}
+
+export interface TowerError {
+  readonly id: bigint
+  readonly fiberId: string
+  readonly interrupted: boolean
+  readonly prettyPrint: string
+  readonly details: Array<TowerErrorDetail>
+}
+
+export interface TowerMetricSnapshot {
+  readonly name: string
+  readonly type: "counter" | "gauge" | "histogram" | "summary" | "frequency"
+  readonly value: unknown
+  readonly tags: ReadonlyArray<{ key: string; value: string }>
+  readonly timestamp: number
+}
+
+export type TowerEvent =
+  | { readonly _tag: "SpanStart"; readonly span: TowerSpan }
+  | { readonly _tag: "SpanEnd"; readonly span: TowerSpan }
+  | { readonly _tag: "Log"; readonly log: TowerLog }
+  | { readonly _tag: "Error"; readonly error: TowerError }
+  | { readonly _tag: "MetricsSnapshot"; readonly metrics: Array<TowerMetricSnapshot> }
+  | { readonly _tag: "ProcessSnapshot"; readonly stats: ProcessStats }
+
+export interface FiberContext {
+  readonly spanName: string | undefined
+  readonly traceId: bigint | undefined
+  readonly annotations: Record<string, unknown>
+}
+
+export interface TowerStoreShape {
+  prefix: string
+  readonly sql: SqlClient.SqlClient
+  readonly events: PubSub.PubSub<TowerEvent>
+  readonly spanCapacity: number
+  readonly logCapacity: number
+  readonly errorCapacity: number
+  metrics: Array<TowerMetricSnapshot>
+  process: ProcessStats | undefined
+}
+
+export function fiberIdCounter(): number {
+  const counter = GlobalValue.globalValue(Symbol.for("effect/Fiber/Id/_fiberCounter"), () =>
+    MutableRef.make(0),
+  )
+  return MutableRef.get(counter)
+}
+
+export class TowerStore extends Context.Tag("effect-start/TowerStore")<
+  TowerStore,
+  TowerStoreShape
+>() {}
+
+export interface TowerStoreOptions {
+  readonly spanCapacity?: number
+  readonly logCapacity?: number
+  readonly errorCapacity?: number
+}
+
+const DDL = [
+  `CREATE TABLE IF NOT EXISTS Span (
+    spanId INTEGER PRIMARY KEY,
+    traceId INTEGER NOT NULL,
+    fiberId TEXT,
+    name TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    parentSpanId INTEGER,
+    startTime TEXT NOT NULL,
+    endTime TEXT,
+    durationMs REAL,
+    status TEXT NOT NULL,
+    attributes TEXT NOT NULL,
+    events TEXT NOT NULL
+  )`,
+  `CREATE TABLE IF NOT EXISTS Log (
+    id INTEGER PRIMARY KEY,
+    level TEXT NOT NULL,
+    message TEXT NOT NULL,
+    fiberId TEXT NOT NULL,
+    cause TEXT,
+    spans TEXT NOT NULL,
+    annotations TEXT NOT NULL
+  )`,
+  `CREATE TABLE IF NOT EXISTS Error (
+    id INTEGER PRIMARY KEY,
+    fiberId TEXT NOT NULL,
+    interrupted INTEGER NOT NULL,
+    prettyPrint TEXT NOT NULL,
+    details TEXT NOT NULL
+  )`,
+  `CREATE TABLE IF NOT EXISTS Fiber (
+    id TEXT PRIMARY KEY,
+    parentId TEXT,
+    spanName TEXT,
+    traceId INTEGER,
+    annotations TEXT NOT NULL
+  )`,
+]
+
+export function layer(
+  options?: TowerStoreOptions,
+): Layer.Layer<TowerStore, SqlClient.SqlError, SqlClient.SqlClient> {
+  return Layer.effect(
+    TowerStore,
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient
+      for (const ddl of DDL) {
+        yield* sql.unsafe(ddl)
+      }
+      store = {
+        ...store,
+        sql,
+        spanCapacity: options?.spanCapacity ?? 1000,
+        logCapacity: options?.logCapacity ?? 5000,
+        errorCapacity: options?.errorCapacity ?? 1000,
+      }
+      return store
+    }),
+  )
+}
+
+function serializeBigint(value: unknown): unknown {
+  if (typeof value === "bigint") return `__bigint__${value}`
+  if (Array.isArray(value)) return value.map(serializeBigint)
+  if (value !== null && typeof value === "object") {
+    const out: Record<string, unknown> = {}
+    for (const [k, v] of Object.entries(value)) {
+      out[k] = serializeBigint(v)
+    }
+    return out
+  }
+  return value
+}
+
+function reviveBigint(value: unknown): unknown {
+  if (typeof value === "string" && value.startsWith("__bigint__")) return BigInt(value.slice(10))
+  if (Array.isArray(value)) return value.map(reviveBigint)
+  if (value !== null && typeof value === "object") {
+    const out: Record<string, unknown> = {}
+    for (const [k, v] of Object.entries(value)) {
+      out[k] = reviveBigint(v)
+    }
+    return out
+  }
+  return value
+}
+
+export const SpanRow = Schema.Struct({
+  spanId: Schema.Number,
+  traceId: Schema.Number,
+  fiberId: Schema.NullOr(Schema.String),
+  name: Schema.String,
+  kind: Schema.String,
+  parentSpanId: Schema.NullOr(Schema.Number),
+  startTime: Schema.String,
+  endTime: Schema.NullOr(Schema.String),
+  durationMs: Schema.NullOr(Schema.Number),
+  status: Schema.String,
+  attributes: Schema.String,
+  events: Schema.String,
+})
+type SpanRow = typeof SpanRow.Type
+
+export const LogRow = Schema.Struct({
+  id: Schema.Number,
+  level: Schema.String,
+  message: Schema.String,
+  fiberId: Schema.String,
+  cause: Schema.NullOr(Schema.String),
+  spans: Schema.String,
+  annotations: Schema.String,
+})
+type LogRow = typeof LogRow.Type
+
+export const ErrorRow = Schema.Struct({
+  id: Schema.Number,
+  fiberId: Schema.String,
+  interrupted: Schema.Number,
+  prettyPrint: Schema.String,
+  details: Schema.String,
+})
+type ErrorRow = typeof ErrorRow.Type
+
+export const FiberRow = Schema.Struct({
+  id: Schema.String,
+  parentId: Schema.NullOr(Schema.String),
+  spanName: Schema.NullOr(Schema.String),
+  traceId: Schema.NullOr(Schema.Number),
+  annotations: Schema.String,
+})
+type FiberRow = typeof FiberRow.Type
+
+function deserializeSpan(row: SpanRow): TowerSpan {
+  const events = reviveBigint(JSON.parse(row.events)) as TowerSpan["events"]
+  return {
+    spanId: BigInt(row.spanId),
+    traceId: BigInt(row.traceId),
+    fiberId: row.fiberId ?? undefined,
+    name: row.name,
+    kind: row.kind,
+    parentSpanId: row.parentSpanId != null ? BigInt(row.parentSpanId) : undefined,
+    startTime: BigInt(row.startTime),
+    endTime: row.endTime ? BigInt(row.endTime) : undefined,
+    durationMs: row.durationMs ?? undefined,
+    status: row.status as TowerSpan["status"],
+    attributes: JSON.parse(row.attributes),
+    events,
+  }
+}
+
+function deserializeLog(row: LogRow): TowerLog {
+  return {
+    id: BigInt(row.id),
+    level: row.level as TowerLog["level"],
+    message: row.message,
+    fiberId: row.fiberId,
+    cause: row.cause ?? undefined,
+    spans: JSON.parse(row.spans),
+    annotations: JSON.parse(row.annotations),
+  }
+}
+
+function deserializeError(row: ErrorRow): TowerError {
+  return {
+    id: BigInt(row.id),
+    fiberId: row.fiberId,
+    interrupted: row.interrupted === 1,
+    prettyPrint: row.prettyPrint,
+    details: JSON.parse(row.details),
+  }
+}
+
+export function insertSpan(sql: SqlClient.SqlClient, span: TowerSpan) {
+  return sql`INSERT INTO Span ${sql({
+    spanId: span.spanId,
+    traceId: span.traceId,
+    fiberId: span.fiberId ?? null,
+    name: span.name,
+    kind: span.kind,
+    parentSpanId: span.parentSpanId ?? null,
+    startTime: span.startTime.toString(),
+    endTime: span.endTime?.toString() ?? null,
+    durationMs: span.durationMs ?? null,
+    status: span.status,
+    attributes: JSON.stringify(span.attributes),
+    events: JSON.stringify(serializeBigint(span.events)),
+  })}`
+}
+
+export function updateSpan(sql: SqlClient.SqlClient, span: TowerSpan) {
+  return sql`UPDATE Span SET
+    endTime = ${span.endTime?.toString() ?? null},
+    durationMs = ${span.durationMs ?? null},
+    status = ${span.status},
+    attributes = ${JSON.stringify(span.attributes)},
+    events = ${JSON.stringify(serializeBigint(span.events))}
+    WHERE spanId = ${span.spanId}`
+}
+
+export function insertLog(sql: SqlClient.SqlClient, log: TowerLog) {
+  return sql`INSERT INTO Log ${sql({
+    id: log.id,
+    level: log.level,
+    message: log.message,
+    fiberId: log.fiberId,
+    cause: log.cause ?? null,
+    spans: JSON.stringify(log.spans),
+    annotations: JSON.stringify(log.annotations),
+  })}`
+}
+
+export function insertError(sql: SqlClient.SqlClient, error: TowerError) {
+  return sql`INSERT INTO Error ${sql({
+    id: error.id,
+    fiberId: error.fiberId,
+    interrupted: error.interrupted ? 1 : 0,
+    prettyPrint: error.prettyPrint,
+    details: JSON.stringify(error.details),
+  })}`
+}
+
+export function upsertFiber(
+  sql: SqlClient.SqlClient,
+  id: string,
+  parentId: string | undefined,
+  spanName: string | undefined,
+  traceId: bigint | undefined,
+  annotations: Record<string, unknown>,
+) {
+  return sql`INSERT OR REPLACE INTO Fiber ${sql({
+    id,
+    parentId: parentId ?? null,
+    spanName: spanName ?? null,
+    traceId: traceId ?? null,
+    annotations: JSON.stringify(annotations),
+  })}`
+}
+
+export function evict(sql: SqlClient.SqlClient, table: string, capacity: number) {
+  return Effect.gen(function* () {
+    const [{ cnt }] = yield* sql<{ cnt: number }>`SELECT count(*) as cnt FROM ${sql(table)}`
+    if (cnt > capacity) {
+      const excess = cnt - capacity
+      yield* sql`DELETE FROM ${sql(table)} WHERE rowid IN (SELECT rowid FROM ${sql(table)} ORDER BY rowid LIMIT ${excess})`
+    }
+  })
+}
+
+export function runWrite(effect: Effect.Effect<unknown, SqlClient.SqlError>) {
+  writeQueue = writeQueue
+    .then(() => Effect.runPromise(Effect.withTracerEnabled(effect, false)).then(() => undefined))
+    .catch(() => undefined)
+}
+
+let writeQueue = Promise.resolve<void>(undefined)
+
+const noTrace = <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<A, E> =>
+  Effect.withTracerEnabled(effect, false)
+
+export function allSpans(sql: SqlClient.SqlClient) {
+  return noTrace(
+    Effect.map(sql<SpanRow>`SELECT * FROM Span ORDER BY rowid`, (rows) =>
+      rows.map(deserializeSpan),
+    ),
+  )
+}
+
+export function allLogs(sql: SqlClient.SqlClient) {
+  return noTrace(
+    Effect.map(sql<LogRow>`SELECT * FROM Log ORDER BY rowid`, (rows) => rows.map(deserializeLog)),
+  )
+}
+
+export function allErrors(sql: SqlClient.SqlClient) {
+  return noTrace(
+    Effect.map(sql<ErrorRow>`SELECT * FROM Error ORDER BY rowid`, (rows) =>
+      rows.map(deserializeError),
+    ),
+  )
+}
+
+export function spansByTraceId(sql: SqlClient.SqlClient, traceId: bigint) {
+  return noTrace(
+    Effect.map(sql<SpanRow>`SELECT * FROM Span WHERE traceId = ${traceId} ORDER BY rowid`, (rows) =>
+      rows.map(deserializeSpan),
+    ),
+  )
+}
+
+export function spansByFiberId(sql: SqlClient.SqlClient, fiberId: string) {
+  return noTrace(
+    Effect.map(sql<SpanRow>`SELECT * FROM Span WHERE fiberId = ${fiberId} ORDER BY rowid`, (rows) =>
+      rows.map(deserializeSpan),
+    ),
+  )
+}
+
+export function logsByFiberId(sql: SqlClient.SqlClient, fiberId: string) {
+  return noTrace(
+    Effect.map(sql<LogRow>`SELECT * FROM Log WHERE fiberId = ${fiberId} ORDER BY rowid`, (rows) =>
+      rows.map(deserializeLog),
+    ),
+  )
+}
+
+export function getFiber(sql: SqlClient.SqlClient, fiberId: string) {
+  return noTrace(
+    Effect.map(sql<FiberRow>`SELECT * FROM Fiber WHERE id = ${fiberId}`, (rows) =>
+      rows.length > 0 ? rows[0] : undefined,
+    ),
+  )
+}
+
+export function getParentChain(sql: SqlClient.SqlClient, fiberId: string) {
+  return noTrace(
+    Effect.gen(function* () {
+      const chain: Array<string> = []
+      const visited = new Set<string>()
+      let current = fiberId
+      while (true) {
+        const rows = yield* sql<FiberRow>`SELECT * FROM Fiber WHERE id = ${current}`
+        if (rows.length === 0 || !rows[0].parentId) break
+        const parentId = rows[0].parentId
+        if (visited.has(parentId)) break
+        chain.push(parentId)
+        visited.add(parentId)
+        current = parentId
+      }
+      return chain.reverse()
+    }),
+  )
+}
+
+export function getFiberContext(sql: SqlClient.SqlClient, fiberId: string) {
+  return noTrace(
+    Effect.map(
+      sql<FiberRow>`SELECT * FROM Fiber WHERE id = ${fiberId}`,
+      (rows): FiberContext | undefined =>
+        rows.length > 0
+          ? {
+              spanName: rows[0].spanName ?? undefined,
+              traceId: rows[0].traceId != null ? BigInt(rows[0].traceId) : undefined,
+              annotations: JSON.parse(rows[0].annotations),
+            }
+          : undefined,
+    ),
+  )
+}

package/src/tower/TowerTracer.ts

@@ -0,0 +1,128 @@
+import * as Effect from "effect/Effect"
+import * as Exit from "effect/Exit"
+import * as Fiber from "effect/Fiber"
+import * as FiberId from "effect/FiberId"
+import * as Layer from "effect/Layer"
+import * as Option from "effect/Option"
+import * as PubSub from "effect/PubSub"
+import * as Tracer from "effect/Tracer"
+import * as TowerStore from "./TowerStore.ts"
+
+const publish = (store: TowerStore.TowerStoreShape, event: TowerStore.TowerEvent) =>
+  Effect.runSync(PubSub.publish(store.events, event))
+
+const fromTracerId = (id: string): bigint | undefined => {
+  try {
+    return BigInt(id)
+  } catch {
+    return undefined
+  }
+}
+
+const make = (store: TowerStore.TowerStoreShape): Tracer.Tracer =>
+  Tracer.make({
+    span(name, parent, context, links, startTime, kind, options) {
+      const parentSpanId =
+        Option.isSome(parent) && parent.value._tag === "Span"
+          ? fromTracerId(parent.value.spanId)
+          : undefined
+      const parentTraceId = Option.isSome(parent) ? fromTracerId(parent.value.traceId) : undefined
+      const traceId = parentTraceId ?? TowerStore.nextTraceId()
+      const spanId = TowerStore.nextSpanId()
+
+      const attributes: Record<string, unknown> = {}
+      const currentFiber = Fiber.getCurrentFiber()
+      const fiberId = Option.isSome(currentFiber)
+        ? FiberId.threadName(currentFiber.value.id())
+        : undefined
+      if (fiberId) {
+        attributes["fiber.id"] = fiberId
+      }
+      if (typeof options?.captureStackTrace === "function") {
+        const stacktrace = options.captureStackTrace()
+        if (stacktrace) {
+          attributes["code.stacktrace"] = stacktrace
+        }
+      }
+
+      const towerSpan: TowerStore.TowerSpan = {
+        spanId,
+        traceId,
+        fiberId,
+        name,
+        kind,
+        parentSpanId,
+        startTime,
+        endTime: undefined,
+        durationMs: undefined,
+        status: "started",
+        attributes,
+        events: [],
+      }
+
+      TowerStore.runWrite(
+        Effect.zipRight(
+          TowerStore.insertSpan(store.sql, towerSpan),
+          TowerStore.evict(store.sql, "Span", store.spanCapacity),
+        ),
+      )
+      publish(store, { _tag: "SpanStart", span: towerSpan })
+
+      const attrs = new Map<string, unknown>(Object.entries(attributes))
+      const spanLinks = [...links]
+
+      const span: Tracer.Span = {
+        _tag: "Span",
+        name,
+        spanId: String(spanId),
+        traceId: String(traceId),
+        parent,
+        context,
+        get status(): Tracer.SpanStatus {
+          if (towerSpan.endTime != null) {
+            return {
+              _tag: "Ended",
+              startTime: towerSpan.startTime,
+              endTime: towerSpan.endTime,
+              exit: Exit.void,
+            }
+          }
+          return { _tag: "Started", startTime: towerSpan.startTime }
+        },
+        attributes: attrs,
+        links: spanLinks,
+        sampled: true,
+        kind,
+        end(endTime, exit) {
+          towerSpan.endTime = endTime
+          towerSpan.durationMs = Number(endTime - towerSpan.startTime) / 1_000_000
+          towerSpan.status = Exit.isSuccess(exit) ? "ok" : "error"
+          TowerStore.runWrite(TowerStore.updateSpan(store.sql, towerSpan))
+          publish(store, { _tag: "SpanEnd", span: towerSpan })
+        },
+        attribute(key, value) {
+          attrs.set(key, value)
+          ;(towerSpan.attributes as Record<string, unknown>)[key] = value
+          TowerStore.runWrite(TowerStore.updateSpan(store.sql, towerSpan))
+        },
+        event(name, startTime, attributes) {
+          towerSpan.events.push({ name, startTime, attributes })
+          TowerStore.runWrite(TowerStore.updateSpan(store.sql, towerSpan))
+        },
+        addLinks(newLinks) {
+          spanLinks.push(...newLinks)
+        },
+      }
+      return span
+    },
+    context(f) {
+      return f()
+    },
+  })
+
+export const layer: Layer.Layer<never, never, TowerStore.TowerStore> = Layer.unwrapEffect(
+  Effect.gen(function* () {
+    const store = yield* TowerStore.TowerStore
+    return Layer.setTracer(make(store))
+  }),
+)

package/src/tower/index.ts

@@ -0,0 +1,3 @@
+export * as Tower from "./Tower.ts"
+export * as TowerStore from "./TowerStore.ts"
+export * as Simulation from "./Simulation.ts"