goldenmatch 0.1.0

package/src/node/backends/workers.ts

@@ -0,0 +1,212 @@
+/**
+ * workers.ts -- Concurrent and parallel block scoring for Node.
+ *
+ * Python uses ThreadPoolExecutor (rapidfuzz releases the GIL). JS is
+ * single-threaded by default; true parallelism requires `worker_threads`
+ * with serialization overhead that's only worth it for large blocks.
+ *
+ * This module ships two schedulers:
+ *   - `scoreBlocksConcurrent` -- Promise.all batching on the main thread.
+ *     No real parallelism, but zero setup cost and good for small/medium
+ *     block counts.
+ *   - `scoreBlocksParallel`   -- piscina-backed worker pool for true CPU
+ *     parallelism. Optional peer dep; falls back to `scoreBlocksConcurrent`
+ *     when piscina isn't installed.
+ *
+ * Mirrors the shape of `goldenmatch.backends.ray_backend.score_blocks_ray`
+ * from the Python source, but stays inside one Node process.
+ */
+
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+import type {
+  BlockResult,
+  MatchkeyConfig,
+  PairKey,
+  ScoredPair,
+} from "../../core/types.js";
+import { pairKey } from "../../core/cluster.js";
+
+export interface WorkerPoolOptions {
+  /** Max blocks scored concurrently per batch. Defaults to 4. */
+  readonly batchSize?: number;
+}
+
+export interface ParallelWorkerOptions {
+  /** Max worker threads. Defaults to min(8, max(2, blocks.length)). */
+  readonly maxThreads?: number;
+  /** Min worker threads kept warm. Defaults to 1. */
+  readonly minThreads?: number;
+  /** Idle timeout in ms before workers exit. Defaults to 1000. */
+  readonly idleTimeout?: number;
+}
+
+/**
+ * Score blocks with cooperative concurrency.
+ *
+ * - For 0 blocks: returns empty.
+ * - For <= 2 blocks: skips batching overhead and runs sequentially via
+ *   `scoreBlocksSequential` (mirrors Python's small-block fast path).
+ * - Otherwise: schedules blocks in batches of `batchSize`, awaiting each
+ *   batch with `Promise.all` so the event loop can interleave I/O.
+ *
+ * Note: `matchedPairs` is mutated as new pairs are discovered (consistent
+ * with `scoreBlocksSequential`). A frozen snapshot is used per block so
+ * concurrent batches see a stable exclusion set, matching Python's
+ * `score_blocks_parallel` contract.
+ */
+export async function scoreBlocksConcurrent(
+  blocks: readonly BlockResult[],
+  mk: MatchkeyConfig,
+  matchedPairs: Set<PairKey>,
+  options: WorkerPoolOptions = {},
+): Promise<readonly ScoredPair[]> {
+  if (blocks.length === 0) return [];
+
+  // Small-block fast path -- sequential is cheaper than batching.
+  if (blocks.length <= 2) {
+    const { scoreBlocksSequential } = await import("../../core/scorer.js");
+    return scoreBlocksSequential(blocks, mk, matchedPairs);
+  }
+
+  const { findFuzzyMatches } = await import("../../core/scorer.js");
+  const batchSize = options.batchSize ?? 4;
+  const results: ScoredPair[] = [];
+
+  for (let i = 0; i < blocks.length; i += batchSize) {
+    const batch = blocks.slice(i, i + batchSize);
+
+    // Snapshot exclude set per batch so concurrent block scoring is stable.
+    const excludeSnapshot: ReadonlySet<PairKey> = new Set(matchedPairs);
+
+    const batchResults = await Promise.all(
+      batch.map((block) =>
+        Promise.resolve().then(() =>
+          findFuzzyMatches(
+            block.rows,
+            mk,
+            excludeSnapshot,
+            block.preScoredPairs,
+          ),
+        ),
+      ),
+    );
+
+    for (const pairs of batchResults) {
+      for (const p of pairs) {
+        const key = pairKey(p.idA, p.idB);
+        if (matchedPairs.has(key)) continue;
+        matchedPairs.add(key);
+        results.push(p);
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Score blocks in true parallel via piscina worker_threads.
+ *
+ * - For 0 blocks: returns empty.
+ * - For <= 2 blocks: runs sequentially (spinning up workers isn't worth it).
+ * - Otherwise: dispatches each block to a piscina worker that runs
+ *   `findFuzzyMatches` in its own V8 isolate, giving true CPU parallelism.
+ *
+ * Falls back to `scoreBlocksConcurrent` with a console warning if piscina
+ * isn't installed (it's an optional peer dep).
+ *
+ * `matchedPairs` is mutated in place with newly discovered pairs, matching
+ * the contract of `scoreBlocksSequential` / `scoreBlocksConcurrent`.
+ */
+export async function scoreBlocksParallel(
+  blocks: readonly BlockResult[],
+  mk: MatchkeyConfig,
+  matchedPairs: Set<PairKey>,
+  options: ParallelWorkerOptions = {},
+): Promise<readonly ScoredPair[]> {
+  if (blocks.length === 0) return [];
+
+  // Small-block fast path -- worker startup isn't worth it.
+  if (blocks.length <= 2) {
+    const { scoreBlocksSequential } = await import("../../core/scorer.js");
+    return scoreBlocksSequential(blocks, mk, matchedPairs);
+  }
+
+  // Dynamically load piscina so it stays an optional peer dep.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let PiscinaCtor: any;
+  try {
+    const mod = await import("piscina" as string);
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const m = mod as any;
+    PiscinaCtor = m.Piscina ?? m.default ?? m;
+  } catch {
+    console.warn(
+      "piscina not installed; falling back to Promise.all concurrency. " +
+        "Install `piscina` as a peer dep for true worker-thread parallelism.",
+    );
+    return scoreBlocksConcurrent(blocks, mk, matchedPairs);
+  }
+
+  const workerScript = resolveWorkerScript();
+
+  const pool = new PiscinaCtor({
+    filename: workerScript,
+    maxThreads: options.maxThreads ?? Math.max(2, Math.min(8, blocks.length)),
+    minThreads: options.minThreads ?? 1,
+    idleTimeout: options.idleTimeout ?? 1000,
+  });
+
+  try {
+    // Snapshot the exclude set as an array so it serializes across threads.
+    const snapshot = Array.from(matchedPairs);
+
+    const results = (await Promise.all(
+      blocks.map(
+        (block) =>
+          pool.run({ block, mk, matchedPairs: snapshot }) as Promise<{
+            pairs: readonly ScoredPair[];
+          }>,
+      ),
+    )) as { pairs: readonly ScoredPair[] }[];
+
+    const all: ScoredPair[] = [];
+    for (const r of results) {
+      for (const p of r.pairs) {
+        const key = pairKey(p.idA, p.idB);
+        if (matchedPairs.has(key)) continue;
+        matchedPairs.add(key);
+        all.push(p);
+      }
+    }
+    return all;
+  } finally {
+    await pool.destroy();
+  }
+}
+
+/**
+ * Resolve the on-disk path of the compiled worker script.
+ *
+ * tsup is configured to emit `score-worker.js` / `.cjs` alongside this
+ * module in the `dist/node/backends/` directory. In dev (pre-build) the
+ * caller can set `GOLDENMATCH_WORKER_SCRIPT` to point at a custom path
+ * (e.g. a ts-node loader).
+ *
+ * Picks `.js` (ESM) first, falling back to `.cjs`. piscina resolves the
+ * file itself -- we just hand it a path string.
+ */
+function resolveWorkerScript(): string {
+  const override = process.env["GOLDENMATCH_WORKER_SCRIPT"];
+  if (override !== undefined && override.length > 0) return override;
+
+  const here = fileURLToPath(import.meta.url);
+  const dir = dirname(here);
+
+  // Preferred: sibling in the same dist/node/backends/ directory.
+  // tsup emits both .js (ESM) and .cjs depending on the package format;
+  // piscina accepts either.
+  return join(dir, "score-worker.js");
+}

package/src/node/config-file.ts

@@ -0,0 +1,66 @@
+/**
+ * config-file.ts -- YAML config loading/saving from disk.
+ *
+ * Node-only. Uses `createRequire` so the optional `yaml` peer dependency
+ * is resolved lazily without breaking edge-safe ESM builds.
+ */
+
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { createRequire } from "node:module";
+import { parseConfigYaml, configToYaml } from "../core/config/loader.js";
+import type { GoldenMatchConfig } from "../core/types.js";
+
+const require = createRequire(import.meta.url);
+
+interface YamlModule {
+  parse: (s: string) => unknown;
+  stringify: (v: unknown) => string;
+}
+
+function loadYamlModule(): YamlModule {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const mod = require("yaml") as YamlModule;
+    if (typeof mod.parse !== "function" || typeof mod.stringify !== "function") {
+      throw new Error("'yaml' module missing parse/stringify exports");
+    }
+    return mod;
+  } catch (err) {
+    const detail = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `'yaml' package is required for config file I/O. Install: npm install yaml (${detail})`,
+    );
+  }
+}
+
+/**
+ * Load and parse a YAML config file into a typed GoldenMatchConfig.
+ *
+ * @throws if the file cannot be read, `yaml` is not installed, or the
+ *   document does not describe a valid config.
+ */
+export function loadConfigFile(path: string): GoldenMatchConfig {
+  const resolved = resolve(path);
+  if (!existsSync(resolved)) {
+    throw new Error(`Config file not found: ${resolved}`);
+  }
+  const content = readFileSync(resolved, "utf8");
+  const yamlMod = loadYamlModule();
+  return parseConfigYaml(content, yamlMod.parse);
+}
+
+/**
+ * Serialize a GoldenMatchConfig to YAML and write it to disk.
+ * Creates parent directories as needed.
+ */
+export function writeConfigFile(path: string, config: GoldenMatchConfig): void {
+  const resolved = resolve(path);
+  const dir = dirname(resolved);
+  if (dir && dir !== "." && !existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  const yamlMod = loadYamlModule();
+  const yamlStr = configToYaml(config, yamlMod.stringify);
+  writeFileSync(resolved, yamlStr, "utf8");
+}

package/src/node/connectors/base.ts

@@ -0,0 +1,57 @@
+/**
+ * base.ts -- Base connector interface and registry.
+ *
+ * Mirrors goldenmatch.connectors.base from the Python package: a small
+ * abstraction over external data sources (Snowflake, BigQuery, etc.) that
+ * exposes connect/read/close lifecycle and a name-based registry.
+ */
+
+import type { Row } from "../../core/types.js";
+
+export interface ConnectorConfig {
+  readonly [key: string]: unknown;
+}
+
+export interface ConnectorQuery {
+  readonly table: string;
+  readonly columns?: readonly string[];
+  readonly limit?: number;
+}
+
+export interface BaseConnector {
+  readonly name: string;
+  connect(): Promise<void>;
+  read(query: string | ConnectorQuery): Promise<Row[]>;
+  close(): Promise<void>;
+}
+
+export interface ConnectorFactory<C extends ConnectorConfig = ConnectorConfig> {
+  (config: C): BaseConnector;
+}
+
+// ---------------------------------------------------------------------------
+// Registry
+// ---------------------------------------------------------------------------
+
+const registry = new Map<string, ConnectorFactory>();
+
+export function registerConnector(name: string, factory: ConnectorFactory): void {
+  registry.set(name, factory);
+}
+
+export function loadConnector<C extends ConnectorConfig = ConnectorConfig>(
+  name: string,
+  config: C,
+): BaseConnector {
+  const factory = registry.get(name);
+  if (!factory) {
+    throw new Error(
+      `Unknown connector: ${name}. Registered: ${[...registry.keys()].join(", ")}`,
+    );
+  }
+  return factory(config);
+}
+
+export function listConnectors(): readonly string[] {
+  return [...registry.keys()];
+}

package/src/node/connectors/bigquery.ts

@@ -0,0 +1,61 @@
+/**
+ * bigquery.ts -- Google BigQuery connector via the optional `@google-cloud/bigquery` peer dep.
+ */
+
+import { createRequire } from "node:module";
+import type { Row } from "../../core/types.js";
+import type { BaseConnector, ConnectorQuery } from "./base.js";
+
+export interface BigQueryConfig {
+  readonly projectId: string;
+  readonly keyFilename?: string;
+  readonly credentials?: Readonly<Record<string, unknown>>;
+  readonly dataset?: string;
+  readonly location?: string;
+}
+
+export function createBigQueryConnector(config: BigQueryConfig): BaseConnector {
+  const require = createRequire(import.meta.url);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let bq: any;
+  try {
+    bq = require("@google-cloud/bigquery");
+  } catch {
+    throw new Error(
+      "'@google-cloud/bigquery' is required for the BigQuery connector. Install: npm install @google-cloud/bigquery",
+    );
+  }
+
+  const client = new bq.BigQuery({
+    projectId: config.projectId,
+    keyFilename: config.keyFilename,
+    credentials: config.credentials,
+    location: config.location,
+  });
+
+  return {
+    name: "bigquery",
+    async connect() {
+      // No-op: BigQuery client uses REST per-query.
+    },
+    async read(query) {
+      const sql =
+        typeof query === "string"
+          ? query
+          : buildSelect(query, config.dataset);
+      const [rows] = await client.query({ query: sql, location: config.location });
+      return rows as Row[];
+    },
+    async close() {
+      // No persistent connection to tear down.
+    },
+  };
+}
+
+function buildSelect(q: ConnectorQuery, dataset?: string): string {
+  const cols = q.columns?.length ? q.columns.join(",") : "*";
+  const tableRef = `${dataset ? `${dataset}.` : ""}${q.table}`;
+  let sql = `SELECT ${cols} FROM \`${tableRef}\``;
+  if (q.limit) sql += ` LIMIT ${q.limit}`;
+  return sql;
+}

package/src/node/connectors/databricks.ts

@@ -0,0 +1,69 @@
+/**
+ * databricks.ts -- Databricks SQL warehouse connector via the optional `@databricks/sql` peer dep.
+ */
+
+import { createRequire } from "node:module";
+import type { Row } from "../../core/types.js";
+import type { BaseConnector, ConnectorQuery } from "./base.js";
+
+export interface DatabricksConfig {
+  readonly serverHostname: string;
+  readonly httpPath: string;
+  readonly token: string;
+  readonly catalog?: string;
+  readonly schema?: string;
+}
+
+export function createDatabricksConnector(config: DatabricksConfig): BaseConnector {
+  const require = createRequire(import.meta.url);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let db: any;
+  try {
+    db = require("@databricks/sql");
+  } catch {
+    throw new Error(
+      "'@databricks/sql' is required for the Databricks connector. Install: npm install @databricks/sql",
+    );
+  }
+
+  const client = new db.DBSQLClient();
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let session: any = null;
+
+  return {
+    name: "databricks",
+    async connect() {
+      await client.connect({
+        host: config.serverHostname,
+        path: config.httpPath,
+        token: config.token,
+      });
+      session = await client.openSession({
+        initialCatalog: config.catalog,
+        initialSchema: config.schema,
+      });
+    },
+    async read(query) {
+      if (!session) throw new Error("Databricks connector not connected. Call connect() first.");
+      const sql = typeof query === "string" ? query : buildSelect(query);
+      const operation = await session.executeStatement(sql);
+      const rows = await operation.fetchAll();
+      await operation.close();
+      return rows as Row[];
+    },
+    async close() {
+      if (session) {
+        await session.close();
+        session = null;
+      }
+      await client.close();
+    },
+  };
+}
+
+function buildSelect(q: ConnectorQuery): string {
+  const cols = q.columns?.length ? q.columns.map((c) => `\`${c}\``).join(",") : "*";
+  let sql = `SELECT ${cols} FROM ${q.table}`;
+  if (q.limit) sql += ` LIMIT ${q.limit}`;
+  return sql;
+}