@effect-bench/node 0.0.0-stage → 0.0.1

package/README.md

@@ -1,3 +1,88 @@
-# Temporary Holding Version
+# effect-bench
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Benchmarks for [Effect](https://effect.website) code, with JSON output and baseline verdicts built for agents and CI.
+
+Community project, not affiliated with the Effect team.
+
+```ts
+// bench/schema.bench.ts
+import { Bench } from '@effect-bench/core'
+import { Effect } from 'effect'
+
+export default Bench.suite('workflowCompiler', { layer: NodeRegistry.layer })(
+  Bench.params('compile', { nodes: [10, 100, 1000] }, {
+    prepare: ({ nodes }) => Effect.succeed(makeDraft(nodes)),   // untimed
+    run: draft => compile(draft),                                // timed
+  }),
+  Bench.group('decode', { baseline: 'sync' })(
+    Bench.sync('sync', () => decodeSync(lead)),
+    Bench.make('effect', () => decodeEffect(lead)),
+  ),
+  Bench.sync('encode', () => encode(payload), { budget: { p99: '5 millis' } }),
+)
+```
+
+```bash
+effect-bench list
+effect-bench run --save before          # store a baseline
+effect-bench run --compare before --json
+effect-bench check --compare before     # exit 1 on a regression or blown budget
+effect-bench run --rigorous --forks 8   # more processes for a tighter interval
+```
+
+## Writing benches
+
+| Constructor | Measures |
+| --- | --- |
+| `Bench.make(name, effect)` | re-runs one Effect value |
+| `Bench.make(name, ctx => effect)` | builds and runs a fresh Effect per operation |
+| `Bench.sync(name, ctx => value)` | a plain function |
+| `Bench.params(name, matrix, { prepare?, run \| sync })` | one bench per combination, ids like `compile[nodes=100]` |
+| `Bench.group(name, { baseline })(...)` | alternatives, reported as a ratio to the baseline |
+
+`Bench.suite(name, { layer, setup, ...options })` builds `layer` once per suite and runs `setup` in the suite's scope; its result is the `ctx` each bench receives. Benches can only use services the layer provides — anything else is a type error.
+
+Options (suite or bench): `samples`, `minTime`, `warmup`, `memory` (GC + heap delta per sample), `budget: { p50, p99 }`, `tags`, and `check` (validates one result before timing).
+
+## Output
+
+`--json` prints one record per line on stdout: a `result` per bench and a final `summary`. Logs go to stderr.
+
+```json
+{"type":"result","schemaVersion":1,"status":"ok","id":"leadSchema/decode/effect","stats":{"medianNs":858,"p99Ns":943,"opsPerSec":1165161,...},"comparison":{"baseline":"before","change":0.051,"ci":[0.039,0.058],"verdict":"slower","stale":false},"relative":{"baselineId":"leadSchema/decode/sync","ratio":5.1}}
+{"type":"summary","schemaVersion":1,"total":4,"failed":0,"regressions":1,"budgetsExceeded":0,"stale":0,"exitCode":1}
+```
+
+## How verdicts work
+
+The CLI measures each suite in several fresh processes (`--forks`, default 3; 6 with `--rigorous`), interleaved fork by fork so slow drift spreads across suites. Drift between processes — JIT decisions, CPU frequency — is much larger than noise within one run, so each fork's median counts as one observation: the 95% interval for the change is a pooled t-interval on log medians.
+
+`faster`/`slower` only when the whole interval is beyond the noise threshold (`--threshold`, default 3%), `same` when it is inside, `inconclusive` otherwise. On a laptop with 3 forks per side, repeated comparisons of unchanged code gave no false verdicts in 96 tries; a 15% slowdown was flagged every time and a 5% one most of the time (otherwise `inconclusive`). Treat `inconclusive` as "measure more": `--rigorous` or more `--forks`.
+
+`--forks 0` measures in the CLI process, and `Runner.run` does the same when called from code. With a single fork on both sides the interval only sees within-run noise and will overstate confidence.
+
+Exit codes: `0` ok, `1` regression or blown budget, `2` failed bench or error, `3` baseline recorded on another runtime or machine.
+
+## Config
+
+Optional `effect-bench.config.ts` in the working directory:
+
+```ts
+import { BenchConfig } from '@effect-bench/cli'
+
+export default BenchConfig.make({
+  include: ['**/bench/*.bench.ts'],
+  modes: { ci: { samples: 50, minTime: '1 second', forks: 5 } },
+  noiseThreshold: 0.05,
+})
+```
+
+## Packages
+
+- `@effect-bench/core` — DSL, measuring loop, stats, comparison, runner, reporters. No platform imports.
+- `@effect-bench/node` — hrtime clock, GC and heap, environment fingerprint, file baseline store.
+- `@effect-bench/cli` — `effect-bench` command.
+
+## Known limitations
+
+- Bench files are loaded with Node's built-in type stripping, so they must use erasable TypeScript syntax only.

package/dist/src/NodeBench.d.ts

@@ -0,0 +1,9 @@
+import { FileSystem, Layer, Path } from 'effect';
+export declare const clock: Layer.Layer<never, never, never>;
+export declare const memory: Layer.Layer<never, never, never>;
+export declare const environment: Layer.Layer<never, never, never>;
+export declare const baselineStore: (directory?: string) => Layer.Layer<never, never, FileSystem.FileSystem | Path.Path>;
+export declare const layer: (options?: {
+    readonly baselineDirectory?: string;
+}) => Layer.Layer<never, never, never>;
+//# sourceMappingURL=NodeBench.d.ts.map

package/dist/src/NodeBench.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"NodeBench.d.ts","sourceRoot":"","sources":["../../src/NodeBench.ts"],"names":[],"mappings":"AAMA,OAAO,EAAU,UAAU,EAAE,KAAK,EAAU,IAAI,EAAU,MAAM,QAAQ,CAAA;AAExE,eAAO,MAAM,KAAK,kCAIhB,CAAA;AAYF,eAAO,MAAM,MAAM,kCAGhB,CAAA;AAUH,eAAO,MAAM,WAAW,kCAetB,CAAA;AAIF,eAAO,MAAM,aAAa,sFAwBvB,CAAA;AAEH,eAAO,MAAM,KAAK,aAAa;IAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAE,qCAGnE,CAAA"}

package/dist/src/NodeBench.js

@@ -0,0 +1,73 @@
+import { execFileSync } from 'node:child_process';
+import os from 'node:os';
+import { setFlagsFromString } from 'node:v8';
+import { runInNewContext } from 'node:vm';
+import { Baseline, BaselineError, BaselineStore, BenchClock, BenchEnvironment, BenchMemory } from '@effect-bench/core';
+import { NodeServices } from '@effect/platform-node';
+import { Effect, FileSystem, Layer, Option, Path, Schema } from 'effect';
+export const clock = Layer.sync(BenchClock, () => {
+    // Offset from layer creation so the value stays well inside float precision.
+    const origin = process.hrtime.bigint();
+    return { nowNs: () => Number(process.hrtime.bigint() - origin) };
+});
+const exposeGc = () => {
+    if (typeof globalThis.gc === 'function')
+        return globalThis.gc;
+    try {
+        setFlagsFromString('--expose-gc');
+        return runInNewContext('gc');
+    }
+    catch {
+        return undefined;
+    }
+};
+export const memory = Layer.sync(BenchMemory, () => ({
+    gc: exposeGc(),
+    heapUsed: () => process.memoryUsage().heapUsed,
+}));
+const git = (args) => {
+    try {
+        return execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+    }
+    catch {
+        return null;
+    }
+};
+export const environment = Layer.succeed(BenchEnvironment, {
+    fingerprint: Effect.sync(() => {
+        const cpus = os.cpus();
+        const status = git(['status', '--porcelain']);
+        return {
+            runtime: typeof process.versions['bun'] === 'string' ? 'bun' : 'node',
+            runtimeVersion: process.version,
+            platform: process.platform,
+            arch: process.arch,
+            cpu: cpus[0]?.model.trim() ?? 'unknown',
+            cpus: cpus.length,
+            gitSha: git(['rev-parse', 'HEAD']),
+            gitDirty: status === null ? null : status.length > 0,
+        };
+    }),
+});
+const BaselineJson = Schema.fromJsonString(Baseline);
+export const baselineStore = (directory = '.bench/baselines') => Layer.effect(BaselineStore, Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const fileOf = (name) => path.join(directory, `${name}.json`);
+    const failure = (name, message) => (cause) => new BaselineError({ name, message, cause });
+    return {
+        save: baseline => Effect.gen(function* () {
+            const json = yield* Schema.encodeEffect(BaselineJson)(baseline);
+            yield* fs.makeDirectory(directory, { recursive: true });
+            yield* fs.writeFileString(fileOf(baseline.name), json);
+        }).pipe(Effect.mapError(failure(baseline.name, `Could not save baseline "${baseline.name}".`))),
+        load: name => Effect.gen(function* () {
+            if (!(yield* fs.exists(fileOf(name))))
+                return Option.none();
+            const json = yield* fs.readFileString(fileOf(name));
+            return Option.some(yield* Schema.decodeUnknownEffect(BaselineJson)(json));
+        }).pipe(Effect.mapError(failure(name, `Could not load baseline "${name}".`))),
+    };
+}));
+export const layer = (options = {}) => Layer.mergeAll(clock, memory, environment, baselineStore(options.baselineDirectory)).pipe(Layer.provide(NodeServices.layer));
+//# sourceMappingURL=NodeBench.js.map

package/dist/src/NodeBench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NodeBench.js","sourceRoot":"","sources":["../../src/NodeBench.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AACzC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AACtH,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAExE,MAAM,CAAC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;IAC/C,6EAA6E;IAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;IACtC,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAA;AAClE,CAAC,CAAC,CAAA;AAEF,MAAM,QAAQ,GAAG,GAA6B,EAAE;IAC9C,IAAI,OAAO,UAAU,CAAC,EAAE,KAAK,UAAU;QAAE,OAAO,UAAU,CAAC,EAAE,CAAA;IAC7D,IAAI,CAAC;QACH,kBAAkB,CAAC,aAAa,CAAC,CAAA;QACjC,OAAO,eAAe,CAAC,IAAI,CAAe,CAAA;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;IACnD,EAAE,EAAE,QAAQ,EAAE;IACd,QAAQ,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ;CAC/C,CAAC,CAAC,CAAA;AAEH,MAAM,GAAG,GAAG,CAAC,IAA2B,EAAiB,EAAE;IACzD,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACpG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE;IACzD,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;QAC5B,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAA;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAA;QAC7C,OAAO;YACL,OAAO,EAAE,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM;YACrE,cAAc,EAAE,OAAO,CAAC,OAAO;YAC/B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,SAAS;YACvC,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YAClC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;SACrD,CAAA;IACH,CAAC,CAAC;CACH,CAAC,CAAA;AAEF,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAA;AAEpD,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,SAAS,GAAG,kBAAkB,EAAE,EAAE,CAC9D,KAAK,CAAC,MAAM,CACV,aAAa,EACb,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,UAAU,CAAA;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAA;IAC7B,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,OAAO,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,OAAe,EAAE,EAAE,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,IAAI,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;IAElH,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,EAAE,CACf,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAA;YAC/D,KAAK,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YACvD,KAAK,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAA;QACxD,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,4BAA4B,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACjG,IAAI,EAAE,IAAI,CAAC,EAAE,CACX,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAA;YAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;YACnD,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC3E,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,4BAA4B,IAAI,IAAI,CAAC,CAAC,CAAC;KAChF,CAAA;AACH,CAAC,CAAC,CACH,CAAA;AAEH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,OAAO,GAA4C,EAAE,EAAE,EAAE,CAC7E,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,aAAa,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CACvF,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAClC,CAAA"}

package/dist/src/index.d.ts

@@ -0,0 +1,2 @@
+export * as NodeBench from './NodeBench.ts';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAA"}

package/dist/src/index.js

@@ -0,0 +1,2 @@
+export * as NodeBench from './NodeBench.js';
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAA"}

package/package.json

@@ -1,6 +1,30 @@
 {
   "name": "@effect-bench/node",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.0.1",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "default": "./dist/src/index.js"
+    }
+  },
+  "dependencies": {
+    "@effect-bench/core": "0.0.1"
+  },
+  "peerDependencies": {
+    "effect": "^4.1.2",
+    "@effect/platform-node": "^4.1.2"
+  },
+  "devDependencies": {
+    "effect": "4.1.2",
+    "@effect/platform-node": "4.1.2"
+  },
+  "files": [
+    "dist/src",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {}
 }

package/src/NodeBench.ts

@@ -0,0 +1,86 @@
+import { execFileSync } from 'node:child_process'
+import os from 'node:os'
+import { setFlagsFromString } from 'node:v8'
+import { runInNewContext } from 'node:vm'
+import { Baseline, BaselineError, BaselineStore, BenchClock, BenchEnvironment, BenchMemory } from '@effect-bench/core'
+import { NodeServices } from '@effect/platform-node'
+import { Effect, FileSystem, Layer, Option, Path, Schema } from 'effect'
+
+export const clock = Layer.sync(BenchClock, () => {
+  // Offset from layer creation so the value stays well inside float precision.
+  const origin = process.hrtime.bigint()
+  return { nowNs: () => Number(process.hrtime.bigint() - origin) }
+})
+
+const exposeGc = (): (() => void) | undefined => {
+  if (typeof globalThis.gc === 'function') return globalThis.gc
+  try {
+    setFlagsFromString('--expose-gc')
+    return runInNewContext('gc') as () => void
+  } catch {
+    return undefined
+  }
+}
+
+export const memory = Layer.sync(BenchMemory, () => ({
+  gc: exposeGc(),
+  heapUsed: () => process.memoryUsage().heapUsed,
+}))
+
+const git = (args: ReadonlyArray<string>): string | null => {
+  try {
+    return execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()
+  } catch {
+    return null
+  }
+}
+
+export const environment = Layer.succeed(BenchEnvironment, {
+  fingerprint: Effect.sync(() => {
+    const cpus = os.cpus()
+    const status = git(['status', '--porcelain'])
+    return {
+      runtime: typeof process.versions['bun'] === 'string' ? 'bun' : 'node',
+      runtimeVersion: process.version,
+      platform: process.platform,
+      arch: process.arch,
+      cpu: cpus[0]?.model.trim() ?? 'unknown',
+      cpus: cpus.length,
+      gitSha: git(['rev-parse', 'HEAD']),
+      gitDirty: status === null ? null : status.length > 0,
+    }
+  }),
+})
+
+const BaselineJson = Schema.fromJsonString(Baseline)
+
+export const baselineStore = (directory = '.bench/baselines') =>
+  Layer.effect(
+    BaselineStore,
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem
+      const path = yield* Path.Path
+      const fileOf = (name: string) => path.join(directory, `${name}.json`)
+      const failure = (name: string, message: string) => (cause: unknown) => new BaselineError({ name, message, cause })
+
+      return {
+        save: baseline =>
+          Effect.gen(function* () {
+            const json = yield* Schema.encodeEffect(BaselineJson)(baseline)
+            yield* fs.makeDirectory(directory, { recursive: true })
+            yield* fs.writeFileString(fileOf(baseline.name), json)
+          }).pipe(Effect.mapError(failure(baseline.name, `Could not save baseline "${baseline.name}".`))),
+        load: name =>
+          Effect.gen(function* () {
+            if (!(yield* fs.exists(fileOf(name)))) return Option.none()
+            const json = yield* fs.readFileString(fileOf(name))
+            return Option.some(yield* Schema.decodeUnknownEffect(BaselineJson)(json))
+          }).pipe(Effect.mapError(failure(name, `Could not load baseline "${name}".`))),
+      }
+    })
+  )
+
+export const layer = (options: { readonly baselineDirectory?: string } = {}) =>
+  Layer.mergeAll(clock, memory, environment, baselineStore(options.baselineDirectory)).pipe(
+    Layer.provide(NodeServices.layer)
+  )

package/src/index.ts

@@ -0,0 +1 @@
+export * as NodeBench from './NodeBench.ts'