hazo_collect 0.2.1

package/dist/server/index.js

@@ -0,0 +1,592 @@
+// src/server/write-adapter.ts
+import { QueryBuilder, wrapResult } from "hazo_connect/server";
+import { generateRequestId } from "hazo_core";
+function createWriteAdapter(adapter) {
+  return {
+    async write({ plugin, landing, canonical }) {
+      return wrapResult(async () => {
+        const txAdapter = adapter;
+        const exec = txAdapter.transaction ? (fn) => txAdapter.transaction(fn) : (fn) => fn(adapter);
+        return exec(async (tx) => {
+          let landingWritten = 0;
+          let canonicalWritten = 0;
+          for (const row of landing) {
+            const payload = typeof row.payload === "string" ? row.payload : JSON.stringify(row.payload);
+            const record = {
+              id: generateRequestId().slice(4),
+              plugin,
+              run_id: row.run_id,
+              idempotency_key: row.idempotency_key,
+              payload,
+              ...row.window?.since != null ? { window_since: row.window.since } : {},
+              ...row.window?.until != null ? { window_until: row.window.until } : {}
+            };
+            const qb = new QueryBuilder().from("hazo_collect_landing").onConflict(["plugin", "idempotency_key"]).doUpdate();
+            await tx.query(qb, "POST", record);
+            landingWritten++;
+          }
+          if (canonical) {
+            for (const row of canonical.rows) {
+              const qb = new QueryBuilder().from(canonical.table).onConflict(canonical.keyCols).doUpdate();
+              await tx.query(qb, "POST", row);
+              canonicalWritten++;
+            }
+          }
+          return { landingWritten, canonicalWritten };
+        });
+      });
+    }
+  };
+}
+
+// src/manager/index.ts
+import { QueryBuilder as QueryBuilder2, wrapResult as wrapResult2 } from "hazo_connect/server";
+import { generateRequestId as generateRequestId2, getCorrelationId, withContext } from "hazo_core";
+
+// src/validate/inputs.ts
+import { Ajv } from "ajv";
+import { HazoValidationError } from "hazo_core";
+var ajv = new Ajv({ allErrors: true });
+function validateInputs(manifest, payload) {
+  if (!manifest.inputs) {
+    return;
+  }
+  const schemaIsValid = ajv.validateSchema(manifest.inputs);
+  if (!schemaIsValid) {
+    const issues = (ajv.errors ?? []).map((e) => ({
+      path: [e.instancePath || e.schemaPath],
+      message: e.message ?? "invalid schema"
+    }));
+    throw new HazoValidationError({
+      code: "HAZO_COLLECT_MALFORMED_MANIFEST",
+      pkg: "hazo_collect",
+      message: `manifest "${manifest.name}" has an invalid inputs JSON-Schema`,
+      issues
+    });
+  }
+  let validate;
+  try {
+    validate = ajv.compile(manifest.inputs);
+  } catch (err) {
+    throw new HazoValidationError({
+      code: "HAZO_COLLECT_MALFORMED_MANIFEST",
+      pkg: "hazo_collect",
+      message: `manifest "${manifest.name}" inputs schema could not be compiled: ${String(err)}`,
+      issues: []
+    });
+  }
+  const valid = validate(payload);
+  if (!valid) {
+    const issues = (validate.errors ?? []).map((e) => ({
+      path: [e.instancePath],
+      message: e.message ?? "validation failed"
+    }));
+    throw new HazoValidationError({
+      code: "HAZO_COLLECT_INVALID_INPUTS",
+      pkg: "hazo_collect",
+      message: `inputs for plugin "${manifest.name}" failed validation`,
+      issues
+    });
+  }
+}
+
+// src/contract-version.ts
+var CONTRACT_VERSION = "1.0.0";
+
+// src/schema/run-result.ts
+import { z } from "zod";
+import { fromZodValidation } from "hazo_core";
+var RunErrorSchema = z.object({
+  code: z.string(),
+  message: z.string(),
+  retryable: z.boolean().optional(),
+  context: z.unknown().optional()
+});
+var RunResultSchema = z.object({
+  plugin: z.string(),
+  run_id: z.string(),
+  kind: z.enum(["source", "sink"]),
+  started_at: z.string(),
+  completed_at: z.string(),
+  status: z.enum(["success", "partial", "failed"]),
+  records_fetched: z.number().int().min(0),
+  records_written: z.number().int().min(0),
+  watermark: z.string().optional(),
+  errors: z.array(RunErrorSchema).default([]),
+  metrics: z.record(z.number()).optional(),
+  contract_version: z.string().default(CONTRACT_VERSION)
+});
+
+// src/runtime/node.ts
+import { createLogger } from "hazo_core";
+
+// src/sdk/http.ts
+import { fetchWithRequestId, sleep } from "hazo_core";
+var MAX_RETRIES = 3;
+var BASE_DELAY_MS = 500;
+function createHttpHelper(_requestId, fetchFn = fetchWithRequestId) {
+  return async (url, init) => {
+    let lastResponse;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      if (attempt > 0) {
+        const delay = BASE_DELAY_MS * Math.pow(2, attempt - 1) + Math.random() * 100;
+        await sleep(delay);
+      }
+      let response;
+      try {
+        response = await fetchFn(url, init);
+      } catch (err) {
+        if (err && typeof err === "object" && "retryable" in err && err.retryable === false) {
+          throw err;
+        }
+        if (attempt === MAX_RETRIES) throw err;
+        continue;
+      }
+      if (response.ok) return response;
+      const shouldRetry = response.status === 429 || response.status >= 500;
+      if (!shouldRetry) return response;
+      lastResponse = response;
+      if (attempt === MAX_RETRIES) return response;
+    }
+    return lastResponse;
+  };
+}
+
+// src/runtime/node.ts
+var nodeRuntime = {
+  async run(def, envelope, adapter) {
+    const writeAdapter = createWriteAdapter(adapter);
+    const log = createLogger("hazo_collect").child({
+      collector: def.manifest.name,
+      runId: envelope.run_id
+    });
+    const http = createHttpHelper(envelope.run_id);
+    const ctx = {
+      envelope,
+      write: writeAdapter.write.bind(writeAdapter),
+      http,
+      log
+    };
+    try {
+      const outcome = await def.run(ctx);
+      return { ok: true, outcome };
+    } catch (error) {
+      return { ok: false, error };
+    }
+  }
+};
+
+// src/manager/index.ts
+async function buildDefaultSecretsProvider() {
+  try {
+    const { EnvSecretsProvider } = await import("hazo_secure/secrets");
+    return new EnvSecretsProvider("HAZO_COLLECT_SECRET");
+  } catch {
+    return {
+      async resolve(names) {
+        if (names.length === 0) return {};
+        throw new Error(
+          "hazo_secure is not installed; provide a SecretsProvider option to createManager"
+        );
+      },
+      async get(name) {
+        throw new Error(
+          `hazo_secure not installed; cannot resolve secret "${name}"`
+        );
+      }
+    };
+  }
+}
+function nowISO() {
+  return (/* @__PURE__ */ new Date()).toISOString();
+}
+function makeFailedResult(plugin, run_id, kind, started_at, error) {
+  return RunResultSchema.parse({
+    plugin,
+    run_id,
+    kind,
+    started_at,
+    completed_at: nowISO(),
+    status: "failed",
+    records_fetched: 0,
+    records_written: 0,
+    errors: [{ code: error.code, message: error.message, retryable: error.retryable }],
+    contract_version: CONTRACT_VERSION
+  });
+}
+async function readWatermark(adapter, plugin) {
+  const qb = new QueryBuilder2().from("hazo_collect_watermarks").where("plugin", "eq", plugin).limit(1);
+  const rows = await adapter.query(qb);
+  return rows[0]?.watermark;
+}
+async function insertRunRow(adapter, params) {
+  const row = {
+    id: generateRequestId2().slice(4),
+    run_id: params.run_id,
+    plugin: params.plugin,
+    kind: params.kind,
+    status: "running",
+    started_at: params.started_at,
+    correlation_id: params.correlation_id,
+    records_fetched: 0,
+    records_written: 0,
+    errors: "[]",
+    contract_version: CONTRACT_VERSION
+  };
+  const qb = new QueryBuilder2().from("hazo_collect_plugin_runs");
+  await adapter.query(qb, "POST", row);
+}
+async function updateRunRow(adapter, run_id, result) {
+  const qb = new QueryBuilder2().from("hazo_collect_plugin_runs").where("run_id", "eq", run_id);
+  await adapter.query(qb, "PATCH", {
+    status: result.status,
+    completed_at: result.completed_at,
+    records_fetched: result.records_fetched,
+    records_written: result.records_written,
+    watermark: result.watermark ?? null,
+    errors: JSON.stringify(result.errors),
+    metrics: result.metrics ? JSON.stringify(result.metrics) : null,
+    contract_version: result.contract_version ?? null
+  });
+}
+async function upsertWatermark(adapter, plugin, watermark) {
+  const qb = new QueryBuilder2().from("hazo_collect_watermarks").onConflict(["plugin"]).doUpdate();
+  await adapter.query(qb, "POST", {
+    plugin,
+    watermark,
+    updated_at: nowISO()
+  });
+}
+async function upsertHealth(adapter, plugin, status, now) {
+  if (status === "success") {
+    const qb = new QueryBuilder2().from("hazo_collect_plugin_health").onConflict(["plugin"]).doUpdate();
+    await adapter.query(qb, "POST", {
+      plugin,
+      last_success_at: now,
+      last_run_at: now,
+      last_status: "success",
+      consecutive_failures: 0,
+      state: "ok"
+    });
+    return;
+  }
+  const qbRead = new QueryBuilder2().from("hazo_collect_plugin_health").where("plugin", "eq", plugin).limit(1);
+  const rows = await adapter.query(qbRead);
+  const current = rows[0]?.consecutive_failures ?? 0;
+  const upsertRow = {
+    plugin,
+    last_run_at: now,
+    last_status: status,
+    consecutive_failures: current + 1
+  };
+  if (status === "failed") {
+    upsertRow.state = "failed";
+  }
+  const qbUpsert = new QueryBuilder2().from("hazo_collect_plugin_health").onConflict(["plugin"]).doUpdate();
+  await adapter.query(qbUpsert, "POST", upsertRow);
+}
+function createManager(opts) {
+  const { adapter, registry } = opts;
+  const runtime = opts.runtime ?? nodeRuntime;
+  let _secretsProviderPromise;
+  function getSecretsProvider() {
+    if (opts.secrets) return Promise.resolve(opts.secrets);
+    if (!_secretsProviderPromise) {
+      _secretsProviderPromise = buildDefaultSecretsProvider();
+    }
+    return _secretsProviderPromise;
+  }
+  async function _executeRun(entry, envelope) {
+    return wrapResult2(async () => {
+      const started_at = nowISO();
+      await insertRunRow(adapter, {
+        run_id: envelope.run_id,
+        plugin: envelope.plugin,
+        kind: entry.manifest.kind,
+        correlation_id: envelope.correlation_id,
+        started_at
+      });
+      const dispatchResult = await runtime.run(entry.worker, envelope, adapter);
+      let result;
+      if (dispatchResult.ok) {
+        const outcome = dispatchResult.outcome;
+        result = RunResultSchema.parse({
+          plugin: envelope.plugin,
+          run_id: envelope.run_id,
+          kind: entry.manifest.kind,
+          started_at,
+          completed_at: nowISO(),
+          status: outcome.status ?? "success",
+          records_fetched: outcome.records_fetched,
+          records_written: outcome.records_written,
+          watermark: outcome.watermark,
+          errors: [],
+          metrics: outcome.metrics,
+          contract_version: CONTRACT_VERSION
+        });
+      } else {
+        const err = dispatchResult.error;
+        result = RunResultSchema.parse({
+          plugin: envelope.plugin,
+          run_id: envelope.run_id,
+          kind: entry.manifest.kind,
+          started_at,
+          completed_at: nowISO(),
+          status: "failed",
+          records_fetched: 0,
+          records_written: 0,
+          errors: [{
+            code: err?.code ?? "HAZO_COLLECT_WORKER_ERROR",
+            message: err instanceof Error ? err.message : String(err),
+            retryable: err?.retryable
+          }],
+          contract_version: CONTRACT_VERSION
+        });
+      }
+      await updateRunRow(adapter, envelope.run_id, result);
+      if (result.status === "success" && result.watermark) {
+        await upsertWatermark(adapter, envelope.plugin, result.watermark);
+      }
+      await upsertHealth(adapter, envelope.plugin, result.status, nowISO());
+      return result;
+    });
+  }
+  async function runNow(pluginName, runOpts) {
+    const entry = registry.get(pluginName);
+    if (!entry) {
+      const run_id2 = generateRequestId2();
+      return {
+        ok: true,
+        data: makeFailedResult(pluginName, run_id2, "source", nowISO(), {
+          code: "HAZO_COLLECT_PLUGIN_NOT_FOUND",
+          message: `Plugin "${pluginName}" not found in registry`
+        })
+      };
+    }
+    const { manifest } = entry;
+    const inputs = runOpts?.inputs ?? {};
+    try {
+      validateInputs(manifest, inputs);
+    } catch (err) {
+      const run_id2 = generateRequestId2();
+      return {
+        ok: true,
+        data: makeFailedResult(pluginName, run_id2, manifest.kind, nowISO(), {
+          code: err?.code ?? "HAZO_COLLECT_INVALID_INPUTS",
+          message: err instanceof Error ? err.message : String(err)
+        })
+      };
+    }
+    let secrets = {};
+    if (manifest.secrets && manifest.secrets.length > 0) {
+      const provider = await getSecretsProvider();
+      try {
+        secrets = await provider.resolve(manifest.secrets);
+      } catch (err) {
+        const run_id2 = generateRequestId2();
+        return {
+          ok: true,
+          data: makeFailedResult(pluginName, run_id2, manifest.kind, nowISO(), {
+            code: err?.code ?? "HAZO_SECURE_SECRET_NOT_FOUND",
+            message: err instanceof Error ? err.message : String(err)
+          })
+        };
+      }
+    }
+    const run_id = generateRequestId2();
+    const correlation_id = getCorrelationId() ?? run_id;
+    return withContext({ correlationId: correlation_id }, async () => {
+      const watermark = await readWatermark(adapter, pluginName);
+      const envelope = {
+        run_id,
+        correlation_id,
+        plugin: pluginName,
+        inputs,
+        secrets,
+        window: { since: watermark, until: nowISO() },
+        attempt: 1,
+        contract_version: CONTRACT_VERSION
+      };
+      return _executeRun(entry, envelope);
+    });
+  }
+  async function runEnvelope(envelope) {
+    const entry = registry.get(envelope.plugin);
+    if (!entry) {
+      return {
+        ok: true,
+        data: makeFailedResult(envelope.plugin, envelope.run_id, "source", nowISO(), {
+          code: "HAZO_COLLECT_PLUGIN_NOT_FOUND",
+          message: `Plugin "${envelope.plugin}" not found in registry`
+        })
+      };
+    }
+    return withContext(
+      { correlationId: envelope.correlation_id },
+      () => _executeRun(entry, envelope)
+    );
+  }
+  return { runNow, runEnvelope };
+}
+
+// src/registry/index.ts
+import { QueryBuilder as QueryBuilder4, wrapResult as wrapResult3 } from "hazo_connect/server";
+
+// src/schema/manifest.ts
+import { z as z2 } from "zod";
+import { fromZodValidation as fromZodValidation2 } from "hazo_core";
+var RetrySchema = z2.object({
+  max: z2.number().int().min(0).default(3),
+  backoff: z2.enum(["exponential", "linear", "constant"]).default("exponential"),
+  base_ms: z2.number().int().min(0).default(1e3),
+  jitter: z2.boolean().default(true)
+});
+var ManifestSchema = z2.object({
+  name: z2.string().min(1),
+  kind: z2.enum(["source", "sink"]),
+  version: z2.string().min(1),
+  runtime: z2.enum(["node", "python"]),
+  entry: z2.string().min(1),
+  schedule: z2.string().optional(),
+  timezone: z2.string().default("UTC"),
+  timeout_sec: z2.number().int().min(1).default(600),
+  concurrency: z2.number().int().min(1).default(1),
+  retry: RetrySchema.optional(),
+  inputs: z2.record(z2.unknown()).optional(),
+  // JSON-Schema object
+  produces: z2.array(z2.string()).optional(),
+  consumes: z2.array(z2.string()).optional(),
+  secrets: z2.array(z2.string()).optional(),
+  idempotency_key: z2.array(z2.string()).min(1),
+  labels: z2.record(z2.string()).optional()
+});
+function parseManifest(input) {
+  const result = ManifestSchema.safeParse(input);
+  if (!result.success) {
+    throw fromZodValidation2(result.error, {
+      pkg: "hazo_collect",
+      code: "HAZO_COLLECT_INVALID_MANIFEST"
+    });
+  }
+  return result.data;
+}
+
+// src/sdk/index.ts
+import { HazoConfigError } from "hazo_core";
+var _collectors = /* @__PURE__ */ new Map();
+function getCollector(name) {
+  return _collectors.get(name);
+}
+function listCollectors() {
+  return Array.from(_collectors.values());
+}
+
+// src/registry/discovery.ts
+import { readFileSync, readdirSync } from "fs";
+import { resolve } from "path";
+import { pathToFileURL } from "url";
+import { QueryBuilder as QueryBuilder3 } from "hazo_connect/server";
+async function scanFolder(pluginsDir) {
+  const entries = readdirSync(pluginsDir, { withFileTypes: true });
+  const subdirs = entries.filter((e) => e.isDirectory());
+  for (const subdir of subdirs) {
+    const manifestPath = resolve(pluginsDir, subdir.name, "manifest.json");
+    const raw = readFileSync(manifestPath, "utf-8");
+    const json = JSON.parse(raw);
+    const manifest = parseManifest(json);
+    if (manifest.runtime !== "node" || manifest.kind !== "source") {
+      continue;
+    }
+    const entryPath = resolve(pluginsDir, subdir.name, manifest.entry);
+    await import(pathToFileURL(entryPath).href);
+  }
+}
+async function discover(opts) {
+  const pluginsDir = opts.pluginsDir ?? "./plugins";
+  await scanFolder(pluginsDir);
+  const defs = listCollectors();
+  await persistRegistry(
+    opts.adapter,
+    defs.map((def) => ({ manifest: def.manifest, source: "folder" })),
+    { reseed: opts.reseed }
+  );
+  const qb = new QueryBuilder3().from("hazo_collect_plugin_registry").where("valid", "eq", 1);
+  const rows = await opts.adapter.query(qb);
+  const snapshotEntries = [];
+  for (const row of rows) {
+    const parsedManifest = parseManifest(JSON.parse(row.manifest));
+    const worker = getCollector(parsedManifest.name);
+    if (!worker) {
+      continue;
+    }
+    if (parsedManifest.version !== worker.manifest.version) {
+      throw new Error(
+        `Version skew for plugin "${parsedManifest.name}": DB has "${parsedManifest.version}", worker has "${worker.manifest.version}"`
+      );
+    }
+    snapshotEntries.push({ manifest: parsedManifest, worker });
+  }
+  let snapshot = new Map(
+    snapshotEntries.map((e) => [e.manifest.name, e])
+  );
+  return {
+    get(name) {
+      return snapshot.get(name);
+    },
+    list() {
+      return Array.from(snapshot.values());
+    },
+    async refresh() {
+      const refreshed = await discover(opts);
+      snapshot = new Map(refreshed.list().map((e) => [e.manifest.name, e]));
+    }
+  };
+}
+
+// src/registry/index.ts
+function createInMemoryRegistry() {
+  return {
+    get(name) {
+      const def = getCollector(name);
+      if (!def) return void 0;
+      return { manifest: def.manifest, worker: def };
+    },
+    list() {
+      return listCollectors().map((def) => ({ manifest: def.manifest, worker: def }));
+    }
+  };
+}
+async function persistRegistry(adapter, entries, opts) {
+  return wrapResult3(async () => {
+    for (const entry of entries) {
+      let valid = 1;
+      let quarantineReason = null;
+      try {
+        parseManifest(entry.manifest);
+      } catch (err) {
+        valid = 0;
+        quarantineReason = err instanceof Error ? err.message : String(err);
+      }
+      const row = {
+        name: entry.manifest.name,
+        kind: entry.manifest.kind,
+        version: entry.manifest.version,
+        runtime: entry.manifest.runtime,
+        manifest: JSON.stringify(entry.manifest),
+        source: entry.source ?? "folder",
+        valid,
+        quarantine_reason: quarantineReason
+      };
+      const qb = opts?.reseed ? new QueryBuilder4().from("hazo_collect_plugin_registry").onConflict(["name"]).doUpdate() : new QueryBuilder4().from("hazo_collect_plugin_registry").onConflict(["name"]).doNothing();
+      await adapter.query(qb, "POST", row);
+    }
+  });
+}
+export {
+  createInMemoryRegistry,
+  createManager,
+  createWriteAdapter,
+  discover,
+  nodeRuntime,
+  persistRegistry
+};

package/package.json

@@ -0,0 +1,82 @@
+{
+  "name": "hazo_collect",
+  "version": "0.2.1",
+  "description": "Collector-manager engine for the Ocdata platform",
+  "type": "module",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./server": {
+      "types": "./dist/server/index.d.ts",
+      "import": "./dist/server/index.js"
+    },
+    "./sdk": {
+      "types": "./dist/sdk/index.d.ts",
+      "import": "./dist/sdk/index.js",
+      "default": "./dist/sdk/index.js"
+    },
+    "./ddl/sqlite.sql": "./dist/ddl/sqlite.sql",
+    "./ddl/postgres.sql": "./dist/ddl/postgres.sql",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "CHANGE_LOG.md",
+    "SETUP_CHECKLIST.md"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "type-check": "tsc --noEmit",
+    "test": "node --experimental-vm-modules ../node_modules/jest/bin/jest.js --config jest.config.cjs",
+    "dev:test-app": "npm run build && cd test-app && npm run dev",
+    "build:test-app": "npm run build && cd test-app && npm run build"
+  },
+  "dependencies": {
+    "zod": "^3.23.8",
+    "ajv": "^8.17.1"
+  },
+  "peerDependencies": {
+    "hazo_core": "^1.1.0",
+    "hazo_connect": "^3.5.1",
+    "hazo_secure": "^1.3.0",
+    "react": "^18.0.0 || ^19.0.0",
+    "react-dom": "^18.0.0 || ^19.0.0",
+    "hazo_ui": "^3.4.1"
+  },
+  "peerDependenciesMeta": {
+    "hazo_secure": {
+      "optional": true
+    },
+    "react": {
+      "optional": true
+    },
+    "react-dom": {
+      "optional": true
+    },
+    "hazo_ui": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "tsup": "^8.3.0",
+    "typescript": "^5.7.2",
+    "jest": "^30.2.0",
+    "ts-jest": "^29.4.5",
+    "@types/jest": "^30.0.0",
+    "jest-environment-node": "^30.2.0",
+    "@types/node": "^22.10.0",
+    "@types/react": "^19.0.0",
+    "@types/react-dom": "^19.0.0",
+    "hazo_core": "^1.1.0",
+    "hazo_connect": "^3.5.1",
+    "hazo_secure": "^1.3.0"
+  },
+  "keywords": [],
+  "author": "Pubs Abayasiri",
+  "license": "MIT"
+}