effect-gpt 0.1.0

package/tests/ts/support/tensorMatchers.ts

@@ -0,0 +1,96 @@
+/**
+ * Custom tensor assertion helpers for deterministic testing.
+ */
+import { expect } from "bun:test"
+import type { Tensor2D } from "../../../src/tensor/Tensor2D"
+
+const DEFAULT_EPSILON = 1e-5
+
+/**
+ * Asserts that a tensor has the expected shape.
+ */
+export const expectShape = (tensor: Tensor2D, shape: [number, number]): void => {
+  expect(tensor.rows).toBe(shape[0])
+  expect(tensor.cols).toBe(shape[1])
+}
+
+/**
+ * Asserts that two tensors have the same shape and all elements are within epsilon.
+ */
+export const expectClose = (
+  actual: Tensor2D,
+  expected: Tensor2D,
+  epsilon: number = DEFAULT_EPSILON
+): void => {
+  expectShape(actual, [expected.rows, expected.cols])
+
+  for (let i = 0; i < actual.data.length; i++) {
+    const a = actual.data[i]
+    const e = expected.data[i]
+    const diff = Math.abs(a - e)
+    if (diff > epsilon) {
+      const row = Math.floor(i / actual.cols)
+      const col = i % actual.cols
+      throw new Error(
+        `Tensor mismatch at [${row}, ${col}]: got ${a}, expected ${e}, diff ${diff} > epsilon ${epsilon}`
+      )
+    }
+  }
+}
+
+/**
+ * Asserts that all elements of a tensor are within epsilon of a target value.
+ */
+export const expectAllClose = (
+  tensor: Tensor2D,
+  value: number,
+  epsilon: number = DEFAULT_EPSILON
+): void => {
+  for (let i = 0; i < tensor.data.length; i++) {
+    const v = tensor.data[i]
+    const diff = Math.abs(v - value)
+    if (diff > epsilon) {
+      const row = Math.floor(i / tensor.cols)
+      const col = i % tensor.cols
+      throw new Error(
+        `Tensor element at [${row}, ${col}]: got ${v}, expected ~${value}, diff ${diff} > epsilon ${epsilon}`
+      )
+    }
+  }
+}
+
+/**
+ * Asserts that two tensors are NOT equal (at least one element differs by more than epsilon).
+ */
+export const expectNotClose = (
+  actual: Tensor2D,
+  expected: Tensor2D,
+  epsilon: number = DEFAULT_EPSILON
+): void => {
+  if (actual.rows !== expected.rows || actual.cols !== expected.cols) {
+    return // Different shapes means not equal
+  }
+
+  for (let i = 0; i < actual.data.length; i++) {
+    const diff = Math.abs(actual.data[i] - expected.data[i])
+    if (diff > epsilon) {
+      return // Found a difference
+    }
+  }
+
+  throw new Error("Expected tensors to differ, but they are equal within epsilon")
+}
+
+/**
+ * Asserts that a tensor contains finite values (no NaN or Infinity).
+ */
+export const expectFinite = (tensor: Tensor2D): void => {
+  for (let i = 0; i < tensor.data.length; i++) {
+    const v = tensor.data[i]
+    if (!Number.isFinite(v)) {
+      const row = Math.floor(i / tensor.cols)
+      const col = i % tensor.cols
+      throw new Error(`Non-finite value at [${row}, ${col}]: ${v}`)
+    }
+  }
+}

package/tests/ts/support.test.ts

@@ -0,0 +1,165 @@
+/**
+ * Tests for Tier 2 test utilities.
+ */
+import { describe, expect, it } from "bun:test"
+import * as T from "../../src/tensor/Tensor2D"
+import {
+  expectShape,
+  expectClose,
+  expectAllClose,
+  expectNotClose,
+  expectFinite
+} from "./support/tensorMatchers"
+import { runEffect, runEffectFail } from "./support/runEffect"
+import {
+  makeEmbeddings,
+  makeSelfAttention,
+  makeFeedForward,
+  makeLayerNorm,
+  makeTransformerBlock,
+  makeOutputProjection,
+  makeRng
+} from "./support/factories"
+import * as Effect from "effect/Effect"
+import { ShapeError } from "../../src/tensor/ops"
+import { EMBEDDING_DIM, HIDDEN_DIM } from "../../src/config"
+
+describe("tensorMatchers", () => {
+  it("expectShape passes for correct shape", () => {
+    const t = T.zeros(3, 4)
+    expectShape(t, [3, 4])
+  })
+
+  it("expectShape fails for wrong shape", () => {
+    const t = T.zeros(3, 4)
+    expect(() => expectShape(t, [3, 5])).toThrow()
+  })
+
+  it("expectClose passes for identical tensors", () => {
+    const t1 = T.fromArray(2, 2, [1, 2, 3, 4])
+    const t2 = T.fromArray(2, 2, [1, 2, 3, 4])
+    expectClose(t1, t2)
+  })
+
+  it("expectClose passes within epsilon", () => {
+    const t1 = T.fromArray(2, 2, [1.0, 2.0, 3.0, 4.0])
+    const t2 = T.fromArray(2, 2, [1.000001, 2.000001, 3.000001, 4.000001])
+    expectClose(t1, t2)
+  })
+
+  it("expectClose fails beyond epsilon", () => {
+    const t1 = T.fromArray(2, 2, [1, 2, 3, 4])
+    const t2 = T.fromArray(2, 2, [1, 2, 3, 5])
+    expect(() => expectClose(t1, t2)).toThrow()
+  })
+
+  it("expectAllClose passes when all near value", () => {
+    const t = T.fromArray(2, 2, [1.0, 1.000005, 0.999995, 1.0])
+    expectAllClose(t, 1.0)
+  })
+
+  it("expectNotClose passes when tensors differ", () => {
+    const t1 = T.fromArray(2, 2, [1, 2, 3, 4])
+    const t2 = T.fromArray(2, 2, [1, 2, 3, 100])
+    expectNotClose(t1, t2)
+  })
+
+  it("expectNotClose fails when tensors are equal", () => {
+    const t1 = T.fromArray(2, 2, [1, 2, 3, 4])
+    const t2 = T.fromArray(2, 2, [1, 2, 3, 4])
+    expect(() => expectNotClose(t1, t2)).toThrow()
+  })
+
+  it("expectFinite passes for finite values", () => {
+    const t = T.fromArray(2, 2, [1, 2, 3, 4])
+    expectFinite(t)
+  })
+
+  it("expectFinite fails for NaN", () => {
+    const t = T.fromArray(2, 2, [1, NaN, 3, 4])
+    expect(() => expectFinite(t)).toThrow()
+  })
+
+  it("expectFinite fails for Infinity", () => {
+    const t = T.fromArray(2, 2, [1, Infinity, 3, 4])
+    expect(() => expectFinite(t)).toThrow()
+  })
+})
+
+describe("runEffect", () => {
+  it("returns value from successful effect", () => {
+    const effect = Effect.succeed(42)
+    expect(runEffect(effect)).toBe(42)
+  })
+
+  it("throws on failed effect", () => {
+    const effect = Effect.fail(new ShapeError("test error"))
+    expect(() => runEffect(effect)).toThrow()
+  })
+
+  it("runEffectFail returns the error", () => {
+    const effect = Effect.fail(new ShapeError("test error"))
+    const error = runEffectFail(effect)
+    expect(error).toBeInstanceOf(ShapeError)
+    expect(error.message).toBe("test error")
+  })
+})
+
+describe("factories", () => {
+  it("makeEmbeddings produces deterministic weights", () => {
+    const e1 = makeEmbeddings(100, { seed: 42 })
+    const e2 = makeEmbeddings(100, { seed: 42 })
+    expectClose(e1.tokenEmbeddings, e2.tokenEmbeddings)
+    expectClose(e1.positionalEmbeddings, e2.positionalEmbeddings)
+  })
+
+  it("makeEmbeddings with different seeds produces different weights", () => {
+    const e1 = makeEmbeddings(100, { seed: 42 })
+    const e2 = makeEmbeddings(100, { seed: 43 })
+    expectNotClose(e1.tokenEmbeddings, e2.tokenEmbeddings)
+  })
+
+  it("makeSelfAttention produces deterministic weights", () => {
+    const a1 = makeSelfAttention({ seed: 42 })
+    const a2 = makeSelfAttention({ seed: 42 })
+    expectClose(a1.wQ, a2.wQ)
+    expectClose(a1.wK, a2.wK)
+    expectClose(a1.wV, a2.wV)
+  })
+
+  it("makeFeedForward produces deterministic weights", () => {
+    const f1 = makeFeedForward({ seed: 42 })
+    const f2 = makeFeedForward({ seed: 42 })
+    expectClose(f1.w1, f2.w1)
+    expectClose(f1.w2, f2.w2)
+  })
+
+  it("makeLayerNorm initializes correctly", () => {
+    const ln = makeLayerNorm()
+    expectShape(ln.gamma, [1, EMBEDDING_DIM])
+    expectShape(ln.beta, [1, EMBEDDING_DIM])
+    expectAllClose(ln.gamma, 1.0)
+    expectAllClose(ln.beta, 0.0)
+  })
+
+  it("makeTransformerBlock produces deterministic weights", () => {
+    const tb1 = makeTransformerBlock({ seed: 42 })
+    const tb2 = makeTransformerBlock({ seed: 42 })
+    expectClose(tb1.attention.wQ, tb2.attention.wQ)
+    expectClose(tb1.feedForward.w1, tb2.feedForward.w1)
+  })
+
+  it("makeOutputProjection produces deterministic weights", () => {
+    const op1 = makeOutputProjection(100, { seed: 42 })
+    const op2 = makeOutputProjection(100, { seed: 42 })
+    expectClose(op1.wOut, op2.wOut)
+  })
+
+  it("makeRng produces deterministic sequences", () => {
+    const rng1 = makeRng(42)
+    const rng2 = makeRng(42)
+    const seq1 = Array.from({ length: 5 }, () => rng1.next())
+    const seq2 = Array.from({ length: 5 }, () => rng2.next())
+    expect(seq1).toEqual(seq2)
+  })
+})

package/tests/ts/train_loop.test.ts

@@ -0,0 +1,229 @@
+import { describe, test, expect } from "bun:test"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import { makeLLM } from "./support/factories"
+import { expectNotClose } from "./support/tensorMatchers"
+import { CANONICAL_SEED } from "./support/seed"
+import {
+  train,
+  makeLLMLayer,
+  makeTrainingConfigLayer,
+  makePreprocessSettingsLayer
+} from "../../src/training/train"
+import { Embeddings } from "../../src/model/Embeddings"
+import { TransformerBlock } from "../../src/model/TransformerBlock"
+import { OutputProjection } from "../../src/model/OutputProjection"
+import * as T from "../../src/tensor/Tensor2D"
+import { SilentLoggerLive, Logger } from "../../src/services/Logger"
+import { NoOpMetricsLive } from "../../src/services/Metrics"
+import { TestServicesLayer as BaseTestServicesLayer } from "./support/stubs"
+
+const TestServicesLayer = Layer.mergeAll(
+  BaseTestServicesLayer,
+  makePreprocessSettingsLayer({ concurrency: "unbounded", batchSize: 1 })
+)
+
+describe("Train Loop", () => {
+  const tinyVocab = ["hello", "world", "is", "this", "test", "</s>"]
+  const tinyCorpus = ["hello world </s>", "this is </s>", "test world </s>"]
+
+  const createTinyLLM = (seed: number = CANONICAL_SEED) =>
+    makeLLM({ seed, vocabWords: tinyVocab, numTransformerBlocks: 1 })
+
+  test("training mutates embeddings weights", () => {
+    const llm = createTinyLLM()
+    const embeddings = llm.network[0] as Embeddings
+    const tokenEmbeddingsBefore = T.clone(embeddings.tokenEmbeddings)
+
+    Effect.runSync(
+      train(tinyCorpus).pipe(
+        Effect.provide(makeLLMLayer(llm)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+
+    expectNotClose(embeddings.tokenEmbeddings, tokenEmbeddingsBefore)
+  })
+
+  test("training mutates transformer weights", () => {
+    const llm = createTinyLLM()
+    const transformer = llm.network[1] as TransformerBlock
+    const w1Before = T.clone(transformer.feedForward.w1)
+    const wQBefore = T.clone(transformer.attention.wQ)
+
+    Effect.runSync(
+      train(tinyCorpus).pipe(
+        Effect.provide(makeLLMLayer(llm)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+
+    expectNotClose(transformer.feedForward.w1, w1Before)
+    expectNotClose(transformer.attention.wQ, wQBefore)
+  })
+
+  test("training mutates output projection weights", () => {
+    const llm = createTinyLLM()
+    const output = llm.network[2] as OutputProjection
+    const wOutBefore = T.clone(output.wOut)
+
+    Effect.runSync(
+      train(tinyCorpus).pipe(
+        Effect.provide(makeLLMLayer(llm)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+
+    expectNotClose(output.wOut, wOutBefore)
+  })
+
+  test("loss decreases over epochs", async () => {
+    const llm = createTinyLLM()
+    const losses: Array<number> = []
+
+    const makeCaptureLossLogger = () => {
+      const service = {
+        log: (_level: any, _message: string, data?: Record<string, unknown>) => {
+          if (data?.loss !== undefined) {
+            losses.push(data.loss as number)
+          }
+          return Effect.void
+        },
+        debug: () => Effect.void,
+        info: (_message: string, data?: Record<string, unknown>) => {
+          if (data?.loss !== undefined) {
+            losses.push(data.loss as number)
+          }
+          return Effect.void
+        },
+        warn: () => Effect.void,
+        error: () => Effect.void
+      }
+      return Layer.succeed(Logger, service)
+    }
+
+    const program = train(tinyCorpus).pipe(
+      Effect.provide(makeLLMLayer(llm)),
+      Effect.provide(makeTrainingConfigLayer({ epochs: 3, learningRate: 0.01 })),
+      Effect.provide(makeCaptureLossLogger()),
+      Effect.provide(NoOpMetricsLive),
+      Effect.provide(makePreprocessSettingsLayer({ concurrency: "unbounded", batchSize: 1 }))
+    )
+
+    await Effect.runPromise(program)
+
+    expect(losses.length).toBe(3)
+    expect(losses[2]).toBeLessThan(losses[0]!)
+  })
+
+  test("seeded RNG produces deterministic training", () => {
+    const llm1 = createTinyLLM(CANONICAL_SEED)
+    const llm2 = createTinyLLM(CANONICAL_SEED)
+
+    const runOnce = (llm: ReturnType<typeof createTinyLLM>) =>
+      Effect.runSync(
+        train(tinyCorpus).pipe(
+          Effect.provide(makeLLMLayer(llm)),
+          Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+          Effect.provide(TestServicesLayer)
+        )
+      )
+    runOnce(llm1)
+    runOnce(llm2)
+
+    const embeddings1 = llm1.network[0] as Embeddings
+    const embeddings2 = llm2.network[0] as Embeddings
+
+    for (let i = 0; i < embeddings1.tokenEmbeddings.data.length; i++) {
+      expect(embeddings1.tokenEmbeddings.data[i]).toBe(embeddings2.tokenEmbeddings.data[i])
+    }
+  })
+
+  test("empty corpus does not crash", () => {
+    const llm = createTinyLLM()
+    expect(() =>
+      Effect.runSync(
+        train([]).pipe(
+          Effect.provide(makeLLMLayer(llm)),
+          Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+          Effect.provide(TestServicesLayer)
+        )
+      )
+    ).not.toThrow()
+  })
+
+  test("single example corpus trains", () => {
+    const llm = createTinyLLM()
+    const embeddings = llm.network[0] as Embeddings
+    const before = T.clone(embeddings.tokenEmbeddings)
+
+    Effect.runSync(
+      train(["hello world </s>"]).pipe(
+        Effect.provide(makeLLMLayer(llm)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+
+    expectNotClose(embeddings.tokenEmbeddings, before)
+  })
+
+  test("multiple epochs further mutate weights", () => {
+    const llm = createTinyLLM()
+    const embeddings = llm.network[0] as Embeddings
+
+    Effect.runSync(
+      train(tinyCorpus).pipe(
+        Effect.provide(makeLLMLayer(llm)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+    const afterEpoch1 = T.clone(embeddings.tokenEmbeddings)
+
+    Effect.runSync(
+      train(tinyCorpus).pipe(
+        Effect.provide(makeLLMLayer(llm)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.01 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+    expectNotClose(embeddings.tokenEmbeddings, afterEpoch1)
+  })
+
+  test("training with higher learning rate causes larger weight changes", () => {
+    const llm1 = createTinyLLM(CANONICAL_SEED)
+    const llm2 = createTinyLLM(CANONICAL_SEED)
+
+    const embeddings1 = llm1.network[0] as Embeddings
+    const embeddings2 = llm2.network[0] as Embeddings
+    const initial = T.clone(embeddings1.tokenEmbeddings)
+
+    Effect.runSync(
+      train(tinyCorpus).pipe(
+        Effect.provide(makeLLMLayer(llm1)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.001 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+    Effect.runSync(
+      train(tinyCorpus).pipe(
+        Effect.provide(makeLLMLayer(llm2)),
+        Effect.provide(makeTrainingConfigLayer({ epochs: 1, learningRate: 0.1 })),
+        Effect.provide(TestServicesLayer)
+      )
+    )
+
+    let diff1 = 0
+    let diff2 = 0
+    for (let i = 0; i < initial.data.length; i++) {
+      diff1 += Math.abs(embeddings1.tokenEmbeddings.data[i]! - initial.data[i]!)
+      diff2 += Math.abs(embeddings2.tokenEmbeddings.data[i]! - initial.data[i]!)
+    }
+
+    expect(diff2).toBeGreaterThan(diff1)
+  })
+})

package/tests/ts/transformer_block.test.ts

@@ -0,0 +1,72 @@
+import { describe, test, expect } from "bun:test"
+import { runEffect } from "./support/runEffect"
+import { expectShape, expectNotClose, expectFinite } from "./support/tensorMatchers"
+import { makeTransformerBlock } from "./support/factories"
+import * as T from "../../src/tensor/Tensor2D"
+import { EMBEDDING_DIM, HIDDEN_DIM } from "../../src/config"
+
+describe("TransformerBlock", () => {
+  test("forward shape preserved [1, EMBEDDING_DIM]", () => {
+    const block = makeTransformerBlock()
+    const input = T.ones(1, EMBEDDING_DIM)
+    const output = runEffect(block.forward(input))
+    expectShape(output, [1, EMBEDDING_DIM])
+  })
+
+  test("forward shape preserved across sequence lengths", () => {
+    const block = makeTransformerBlock()
+    for (let seqLen = 1; seqLen <= 4; seqLen++) {
+      const input = T.ones(seqLen, EMBEDDING_DIM)
+      const output = runEffect(block.forward(input))
+      expectShape(output, [seqLen, EMBEDDING_DIM])
+    }
+  })
+
+  test("output contains finite values", () => {
+    const block = makeTransformerBlock()
+    const input = T.ones(2, EMBEDDING_DIM)
+    const output = runEffect(block.forward(input))
+    expectFinite(output)
+  })
+
+  test("backward shape matches input", () => {
+    const block = makeTransformerBlock()
+    const input = T.ones(2, EMBEDDING_DIM)
+    runEffect(block.forward(input))
+    const dOut = T.ones(2, EMBEDDING_DIM)
+    const grad = runEffect(block.backward(dOut, 0.01))
+    expectShape(grad, [2, EMBEDDING_DIM])
+  })
+
+  test("backward updates sub-layer weights", () => {
+    const block = makeTransformerBlock()
+    const input = T.zeros(2, EMBEDDING_DIM)
+    for (let i = 0; i < input.data.length; i++) {
+      input.data[i] = (i % 10) * 0.1
+    }
+
+    const wQBefore = T.clone(block.attention.wQ)
+    const w1Before = T.clone(block.feedForward.w1)
+    const betaBefore = T.clone(block.norm1.beta)
+
+    runEffect(block.forward(input))
+    const dOut = T.ones(2, EMBEDDING_DIM)
+    runEffect(block.backward(dOut, 0.01))
+
+    expectNotClose(block.attention.wQ, wQBefore)
+    expectNotClose(block.feedForward.w1, w1Before)
+    expectNotClose(block.norm1.beta, betaBefore)
+  })
+
+  test("parametersCount equals sum of components", () => {
+    const block = makeTransformerBlock()
+
+    const attentionParams = 3 * EMBEDDING_DIM * EMBEDDING_DIM
+    const feedForwardParams =
+      EMBEDDING_DIM * HIDDEN_DIM + HIDDEN_DIM + HIDDEN_DIM * EMBEDDING_DIM + EMBEDDING_DIM
+    const normParams = 2 * (2 * EMBEDDING_DIM)
+
+    const expectedTotal = attentionParams + feedForwardParams + normParams
+    expect(block.parametersCount).toBe(expectedTotal)
+  })
+})

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "lib": ["ES2022"],
+    "types": ["bun-types"],
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsconfig.test.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "types": ["bun-types"],
+    "rootDir": "."
+  },
+  "include": ["src", "tests/ts"]
+}