@segmentstream/ontology 0.0.0-stage → 0.92.1

package/dist/internal/diagnostics.js

@@ -0,0 +1,594 @@
+// ../diagnostics/src/index.ts
+import { randomUUID as randomUUID2 } from "node:crypto";
+
+// ../diagnostics/src/report.ts
+import { createHash, randomUUID } from "node:crypto";
+
+// ../diagnostics/src/redact.ts
+function makeRedactor(values = []) {
+  const secrets = [
+    ...new Set(values.filter((v) => typeof v === "string" && v.length > 0))
+  ].sort((a, b) => b.length - a.length);
+  return (input) => {
+    let text = input;
+    for (const value of secrets) text = text.replaceAll(value, "[REDACTED]");
+    return redactCopyContext(redactPrivateKeys(text)).replace(
+      /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?):\/\/\S+/gi,
+      "[REDACTED CONNECTION URI]"
+    ).replace(
+      /\b(?:sscli_(?:access_|key_)?v1|ssbrowser_(?:link_)?v1)\.[a-f0-9-]{36}\.[A-Za-z0-9_-]{43}\b/gi,
+      "[REDACTED CLI CREDENTIAL]"
+    ).replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9+/_=.:~-]+/gi, "[REDACTED AUTHORIZATION]").replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED JWT]").replace(
+      /\b(?:sk|rk|vercel_blob_rw|vercel_blob_r|access_token|refresh_token)_[A-Za-z0-9._~-]{8,}/g,
+      "[REDACTED]"
+    ).replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, "$1[REDACTED]@").replace(
+      /([?&](?:access_token|api_key|apikey|key|password|refresh_token|secret|token|signature|sig)=)[^&#\s]*/gi,
+      "$1[REDACTED]"
+    ).replace(
+      /((?:["']?)(?:password|passwd|credential|secret|client_secret|api_key|access_token|refresh_token|authorization|cookie|set-cookie|token)(?:["']?)\s*[:=]\s*)(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,;})]+)/gi,
+      "$1[REDACTED]"
+    ).replace(/(Key\s*\([^)]*\)\s*=\s*)\([\s\S]*/gi, (match, prefix) => {
+      const endings = [") already exists", ") is not present"];
+      const end = Math.max(...endings.map((ending) => match.lastIndexOf(ending)));
+      return `${prefix}([REDACTED])${end < 0 ? "" : match.slice(end + 1)}`;
+    }).replace(/(Failing row contains\s+)[\s\S]*/gi, "$1[REDACTED]").replace(/(SQL statement\s+)[\s\S]*/gi, "$1[REDACTED]").replace(/(JSON data, line \d+:\s*)[\s\S]*/gi, "$1[REDACTED]").replace(/((?:Token|Escape sequence|but found)\s+")[\s\S]*/gi, (match, prefix) => {
+      const end = Math.max(match.lastIndexOf('" is invalid'), match.lastIndexOf('".'));
+      return `${prefix}[REDACTED]${end < 0 ? '"' : match.slice(end)}`;
+    }).replace(/(invalid input syntax for [^:]+:\s*)"[^"]*"/gi, "$1[REDACTED]");
+  };
+}
+function redactCopyContext(text) {
+  const lines = /\bCOPY [^\n]*/gi;
+  let line;
+  while ((line = lines.exec(text)) !== null) {
+    const position = line[0].indexOf(", line ");
+    if (position < 0) continue;
+    const header = /^, line \d+(?:, column [^:\n]+)?:\s*/i.exec(line[0].slice(position));
+    return text.slice(0, line.index + (header ? position + header[0].length : 5)) + "[REDACTED]";
+  }
+  return text;
+}
+function environmentSecrets(env) {
+  return Object.entries(env).filter(
+    ([key]) => /(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|API_KEY|COOKIE|SIGNING_KEY|DATABASE_URL|POSTGRES_URL|GRANT)/i.test(
+      key
+    )
+  ).map(([, value]) => value);
+}
+function isSensitiveField(key) {
+  return /password|passwd|credential|secret|token|authorization|cookie|private[_-]?key|api[_-]?key|grant/i.test(
+    key
+  );
+}
+function redactPrivateKeys(text) {
+  const begin = /-----BEGIN [^-]*PRIVATE KEY-----/g;
+  const end = /-----END [^-]*PRIVATE KEY-----/g;
+  const parts = [];
+  let copiedThrough = 0;
+  let opening;
+  while ((opening = begin.exec(text)) !== null) {
+    end.lastIndex = begin.lastIndex;
+    if (end.exec(text) === null) {
+      break;
+    }
+    parts.push(text.slice(copiedThrough, opening.index), "[REDACTED PRIVATE KEY]");
+    copiedThrough = end.lastIndex;
+    begin.lastIndex = copiedThrough;
+  }
+  parts.push(text.slice(copiedThrough));
+  return parts.join("");
+}
+
+// ../diagnostics/src/report.ts
+var failureRedactions = /* @__PURE__ */ new WeakMap();
+var MAX_BYTES = 1024 * 1024;
+var MAX_NODES = 1024;
+var FIELDS = [
+  "fiberId",
+  "code",
+  "status",
+  "statusCode",
+  "provider",
+  "providerCode",
+  "providerRequestId",
+  "method",
+  "host",
+  "path",
+  "retryable",
+  "constraint",
+  "detail",
+  "hint",
+  "table",
+  "column",
+  "where",
+  "position",
+  "sqlState",
+  "field",
+  "expected",
+  "actual",
+  "setting",
+  "resourceId",
+  "runtimeAttemptId",
+  "sandboxId",
+  "sessionId",
+  "remoteErrorId",
+  "endpoint",
+  "requestId",
+  "reason",
+  "connection",
+  "providerMessage",
+  "providerDescription",
+  "stderr",
+  "exitCode"
+];
+var links = Symbol.for("segmentstream.diagnostic.links");
+var incomplete = Symbol.for("segmentstream.diagnostic.omissions");
+var RELATIONS = ["cause", "aggregate", "primary", "cleanup", "concurrent"];
+var MAX_EDGES = 8192;
+function property(value, key) {
+  if ((typeof value !== "object" || value === null) && typeof value !== "function")
+    return void 0;
+  try {
+    return Reflect.get(value, key);
+  } catch {
+    return void 0;
+  }
+}
+function scalar(value) {
+  return typeof value === "string" || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value);
+}
+function effectCause(value) {
+  if (property(value, "~effect/Cause") === "~effect/Cause") {
+    const reasons = property(value, "reasons");
+    if (Array.isArray(reasons))
+      return {
+        type: "EffectCause",
+        message: "Effect execution failed",
+        children: reasons.slice(0, MAX_NODES).map((error) => ({ error, relation: "concurrent" })),
+        limited: reasons.length > MAX_NODES
+      };
+  }
+  const legacy = property(value, Symbol.for("effect/Cause")) !== void 0;
+  if (!legacy && property(value, "~effect/Cause/Reason") !== "~effect/Cause/Reason")
+    return void 0;
+  const type = property(value, "_tag");
+  switch (type) {
+    case "Fail":
+      return {
+        type,
+        message: "Typed failure",
+        children: [{ error: property(value, "error"), relation: "cause" }]
+      };
+    case "Die":
+      return {
+        type,
+        message: "Unexpected defect",
+        children: [{ error: property(value, "defect"), relation: "cause" }]
+      };
+    case "Interrupt":
+      return { type, message: "Fiber interrupted", children: [] };
+    case "Empty":
+      return legacy ? { type, message: "Empty Effect cause", children: [] } : void 0;
+    case "Sequential":
+    case "Parallel":
+      return legacy ? {
+        type,
+        message: `${type} Effect causes`,
+        children: ["left", "right"].map((key) => ({
+          error: property(value, key),
+          relation: type === "Parallel" ? "concurrent" : "cause"
+        }))
+      } : void 0;
+    default:
+      return void 0;
+  }
+}
+function ownValues(value) {
+  if (!value || typeof value !== "object") return [];
+  try {
+    return Object.values(Object.getOwnPropertyDescriptors(value)).filter((descriptor) => "value" in descriptor).map((descriptor) => descriptor.value);
+  } catch {
+    return [];
+  }
+}
+function schemaInputRedactions(root) {
+  const values = /* @__PURE__ */ new Set();
+  const seen = /* @__PURE__ */ new Set();
+  const errors = [root];
+  const issues = [];
+  let limited = false;
+  let traversed = 0;
+  while (errors.length && seen.size < MAX_NODES) {
+    if (++traversed > MAX_EDGES || errors.length > MAX_EDGES) {
+      limited = true;
+      break;
+    }
+    const value = errors.shift();
+    if (seen.has(value)) continue;
+    seen.add(value);
+    if (value !== null && (typeof value === "object" || typeof value === "function")) {
+      for (const secret of failureRedactions.get(value) ?? []) {
+        if (secret) values.add(secret);
+      }
+    }
+    if (property(value, "_tag") === "SchemaError" || property(value, "name") === "SchemaError")
+      issues.push(property(value, "issue"));
+    const effect = effectCause(value);
+    if (effect?.limited) limited = true;
+    if (effect) errors.push(...effect.children.map((child) => child.error));
+    const explicit = property(value, links);
+    if (Array.isArray(explicit)) {
+      if (explicit.length > MAX_NODES) limited = true;
+      errors.push(...explicit.slice(0, MAX_NODES).map((item) => property(item, "error")));
+    }
+    const cause = property(value, "cause");
+    if (cause !== void 0) errors.push(cause);
+    const aggregate = property(value, "errors");
+    if (property(value, "name") === "AggregateError" && Array.isArray(aggregate)) {
+      if (aggregate.length > MAX_NODES) limited = true;
+      errors.push(...aggregate.slice(0, MAX_NODES));
+    }
+  }
+  if (errors.length) limited = true;
+  seen.clear();
+  const inputs = [];
+  while (issues.length && seen.size < MAX_NODES) {
+    if (++traversed > MAX_EDGES || issues.length > MAX_EDGES) {
+      limited = true;
+      break;
+    }
+    const issue = issues.pop();
+    if (!issue || seen.has(issue)) continue;
+    seen.add(issue);
+    for (const key of ["input", "actual"]) {
+      const input = property(issue, key);
+      if (input !== void 0) inputs.push(input);
+    }
+    const child = property(issue, "issue");
+    if (child !== void 0) issues.push(child);
+    const children = property(issue, "issues");
+    if (Array.isArray(children)) {
+      if (children.length > MAX_NODES) limited = true;
+      issues.push(...children.slice(0, MAX_NODES));
+    }
+  }
+  if (issues.length) limited = true;
+  seen.clear();
+  let characters = 0;
+  while (inputs.length && seen.size < 4096 && characters < MAX_BYTES) {
+    if (++traversed > MAX_EDGES || inputs.length > MAX_EDGES) {
+      limited = true;
+      break;
+    }
+    const value = inputs.pop();
+    if (seen.has(value)) continue;
+    seen.add(value);
+    if (scalar(value) && String(value)) {
+      const text = String(value);
+      characters += text.length;
+      values.add(text);
+      if (typeof value === "string") values.add(JSON.stringify(value).slice(1, -1));
+    } else {
+      const children = ownValues(value);
+      if (children.length > 4096) limited = true;
+      inputs.push(...children.slice(0, 4096));
+    }
+  }
+  if (inputs.length) limited = true;
+  return { values: [...values], limited };
+}
+function createFailureReport(error, options) {
+  const schemaInputs = schemaInputRedactions(error);
+  if (schemaInputs.limited) return redactionLimitReport();
+  const redact = makeRedactor([...options.redactedValues ?? [], ...schemaInputs.values]);
+  let remaining = MAX_BYTES - 4096;
+  const omissions = /* @__PURE__ */ new Set();
+  const clean = (value) => {
+    const safe = redact(value.slice(0, MAX_BYTES));
+    if (value.length > MAX_BYTES) omissions.add("field input exceeded 1 MiB");
+    const bytes = Buffer.from(JSON.stringify(safe));
+    const allowance = Math.max(0, remaining);
+    if (bytes.length > allowance) {
+      omissions.add("report exceeded 1 MiB");
+      remaining = 0;
+      let low = 0;
+      let high = safe.length;
+      while (low < high) {
+        const middle = Math.ceil((low + high) / 2);
+        if (Buffer.byteLength(JSON.stringify(safe.slice(0, middle))) + 20 <= allowance)
+          low = middle;
+        else high = middle - 1;
+      }
+      return safe.slice(0, low).replace(/[\uD800-\uDBFF]$/, "") + " [TRUNCATED]";
+    }
+    remaining -= bytes.length;
+    return safe;
+  };
+  const report = {
+    version: 1,
+    errorId: options.errorId && /^[A-Za-z0-9_.:-]{1,200}$/.test(options.errorId) ? options.errorId : `err_${randomUUID().replaceAll("-", "")}`,
+    timestamp: options.timestamp?.slice(0, 100) ?? (/* @__PURE__ */ new Date()).toISOString(),
+    service: clean(options.service),
+    context: {},
+    roots: [],
+    nodes: [],
+    complete: true,
+    omissions: []
+  };
+  const contextEntries = Object.entries(options.context ?? {});
+  if (contextEntries.length > 128) omissions.add("context exceeded 128 fields");
+  for (const [key, value] of contextEntries.slice(0, 128)) {
+    if (scalar(value))
+      report.context[clean(key)] = isSensitiveField(key) ? "[REDACTED]" : typeof value === "string" ? clean(value) : value;
+  }
+  const seen = /* @__PURE__ */ new Map();
+  const queue = [];
+  const add = (value) => {
+    const object = typeof value === "object" && value !== null || typeof value === "function";
+    if (object && seen.has(value)) return seen.get(value);
+    if (queue.length >= MAX_NODES || remaining < 1024) {
+      omissions.add(
+        queue.length >= MAX_NODES ? "report exceeded 1024 nodes" : "report exceeded 1 MiB"
+      );
+      return void 0;
+    }
+    const id = `n${queue.length + 1}`;
+    if (object) seen.set(value, id);
+    queue.push({ value, id });
+    remaining -= 512;
+    return id;
+  };
+  const root = add(error);
+  if (root) report.roots.push(root);
+  let edges = 0;
+  for (let index = 0; index < queue.length; index++) {
+    const { value, id } = queue[index];
+    const inheritedOmissions = property(value, incomplete);
+    if (Array.isArray(inheritedOmissions)) {
+      for (const reason of inheritedOmissions.slice(0, 128)) {
+        if (typeof reason === "string") omissions.add(clean(reason));
+      }
+    }
+    const effect = effectCause(value);
+    const name = property(value, "_tag") ?? property(value, "name") ?? effect?.type;
+    const message = property(value, "message") ?? effect?.message;
+    const node = {
+      id,
+      type: typeof name === "string" ? clean(name) : "NonErrorThrow",
+      message: clean(
+        typeof message === "string" && message ? message : typeof value === "string" ? value : `Thrown ${typeof value} value`
+      ),
+      causes: []
+    };
+    for (const key of ["stack", "operation", "diagnostic"]) {
+      const field = property(value, key);
+      if (typeof field === "string") node[key] = clean(field);
+    }
+    const details = {};
+    for (const key of FIELDS) {
+      const field = property(value, key);
+      if (scalar(field))
+        details[key] = (key === "expected" || key === "actual") && isSensitiveField(String(property(value, "field") ?? "")) ? "[REDACTED]" : typeof field === "string" ? clean(field) : field;
+    }
+    if (Object.keys(details).length) node.details = details;
+    const children = effect?.children ?? [];
+    if (effect?.limited) omissions.add("Effect cause exceeded 1024 reasons");
+    const explicit = property(value, links);
+    if (Array.isArray(explicit)) {
+      for (const item of explicit.slice(0, MAX_NODES)) {
+        const relation = property(item, "relation");
+        if (typeof relation === "string" && RELATIONS.includes(relation))
+          children.push({ error: property(item, "error"), relation });
+        else omissions.add("invalid diagnostic cause link");
+      }
+      if (explicit.length > MAX_NODES) omissions.add("cause links exceeded 1024 children");
+    }
+    const cause = property(value, "cause");
+    if (cause !== void 0) children.push({ error: cause, relation: "cause" });
+    if (value instanceof AggregateError || property(value, "name") === "AggregateError") {
+      const errors = property(value, "errors");
+      if (Array.isArray(errors)) {
+        for (const item of errors.slice(0, MAX_NODES))
+          children.push({ error: item, relation: "aggregate" });
+        if (errors.length > MAX_NODES) omissions.add("aggregate exceeded 1024 children");
+      }
+    }
+    for (const child of children) {
+      if (edges >= MAX_EDGES || remaining < 100) {
+        omissions.add(
+          edges >= MAX_EDGES ? "report exceeded 8192 cause links" : "report exceeded 1 MiB"
+        );
+        break;
+      }
+      edges++;
+      remaining -= 100;
+      const childId = add(child.error);
+      if (childId) node.causes.push({ id: childId, relation: child.relation });
+    }
+    report.nodes.push(node);
+  }
+  report.omissions = [...omissions];
+  report.complete = !report.omissions.length;
+  return fitReport(report);
+}
+function redactionLimitReport() {
+  return {
+    version: 1,
+    errorId: `err_${randomUUID().replaceAll("-", "")}`,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    service: "diagnostics",
+    context: {},
+    roots: ["n1"],
+    nodes: [
+      {
+        id: "n1",
+        type: "RedactionLimitError",
+        message: "Failure details omitted because secret-redaction traversal was incomplete.",
+        causes: []
+      }
+    ],
+    complete: false,
+    omissions: [
+      "secret redaction traversal exceeded its defensive budget (1024 graph/issue nodes, 4096 input values, 8192 traversal steps, 1 MiB)"
+    ]
+  };
+}
+function fitReport(report) {
+  const wireBytes = () => Buffer.byteLength(JSON.stringify(report));
+  if (wireBytes() <= MAX_BYTES) return report;
+  report.complete = false;
+  report.omissions.push("serialized report exceeded 1 MiB");
+  while (wireBytes() > MAX_BYTES && report.nodes.length > 1) report.nodes.pop();
+  const retained = new Set(report.nodes.map((node) => node.id));
+  report.roots = report.roots.filter((id) => retained.has(id));
+  for (const node of report.nodes)
+    node.causes = node.causes.filter((edge) => retained.has(edge.id));
+  if (wireBytes() > MAX_BYTES) {
+    report.context = {};
+    report.service = report.service.slice(0, 200);
+    report.omissions = report.omissions.map((message) => message.slice(0, 1e3)).slice(0, 128);
+    report.nodes = report.nodes.map((node) => ({
+      id: node.id,
+      type: node.type.slice(0, 200),
+      message: node.message.slice(0, 1e3) + " [TRUNCATED]",
+      causes: node.causes
+    }));
+  }
+  return report;
+}
+function renderFailure(report, maxBytes = 8192) {
+  const nodes = new Map(report.nodes.map((node) => [node.id, node]));
+  const seen = /* @__PURE__ */ new Set();
+  const pending = report.roots.map((id) => ({ id, depth: 0, relation: "root" }));
+  const lines = [`Failure ${report.errorId} (${report.service})`];
+  while (pending.length) {
+    const item = pending.pop();
+    const prefix = `${"  ".repeat(Math.min(item.depth, 32))}${item.relation}: `;
+    if (seen.has(item.id)) {
+      lines.push(`${prefix}[reference ${item.id}]`);
+      continue;
+    }
+    seen.add(item.id);
+    const node = nodes.get(item.id);
+    if (!node) {
+      lines.push(`${prefix}[missing ${item.id}]`);
+      continue;
+    }
+    lines.push(
+      `${prefix}${node.type}${node.diagnostic ? `:${node.diagnostic}` : ""}${node.operation ? ` at ${node.operation}` : ""}: ${node.message}`
+    );
+    if (node.details) lines.push(`${prefix}${JSON.stringify(node.details)}`);
+    for (const child of [...node.causes].reverse())
+      pending.push({ ...child, depth: item.depth + 1 });
+  }
+  if (!report.complete) lines.push(`Incomplete: ${report.omissions.join("; ")}`);
+  const text = lines.join("\n");
+  if (Buffer.byteLength(text) <= maxBytes) return text;
+  const suffix = `
+[Journal shortened; complete report in diagnostics logs: ${report.errorId}]`;
+  return Buffer.from(text).subarray(0, Math.max(0, maxBytes - Buffer.byteLength(suffix) - 3)).toString("utf8").replace(/\uFFFD$/, "") + suffix;
+}
+function encodeReport(report) {
+  const direct = JSON.stringify({ event: "diagnostics.report", report });
+  if (Buffer.byteLength(direct) <= 4096) return [direct];
+  const bytes = Buffer.from(JSON.stringify(report));
+  const checksum = createHash("sha256").update(bytes).digest("hex");
+  const size = 2e3;
+  const correlation = Object.fromEntries(
+    ["workflowRunId", "requestId", "pipelineRunId"].flatMap(
+      (key) => typeof report.context[key] === "string" && /^[A-Za-z0-9_.:-]{1,200}$/.test(String(report.context[key])) ? [[key, String(report.context[key]).slice(0, 200)]] : []
+    )
+  );
+  const total = Math.ceil(bytes.length / size);
+  return Array.from(
+    { length: total },
+    (_, i) => JSON.stringify({
+      event: "diagnostics.chunk",
+      errorId: report.errorId,
+      part: i + 1,
+      total,
+      checksum,
+      correlation,
+      encoding: "base64",
+      data: bytes.subarray(i * size, (i + 1) * size).toString("base64")
+    })
+  );
+}
+
+// ../diagnostics/src/index.ts
+function makeErrorReporter({
+  configuration,
+  generateId = randomUUID2,
+  now = () => /* @__PURE__ */ new Date(),
+  observe,
+  write = (severity, line) => severity === "warning" ? console.warn(line) : console.error(line)
+}) {
+  function capture(error, context) {
+    let report;
+    const errorId = context.errorReference ?? `err_${randomUUID2().replaceAll("-", "")}`;
+    try {
+      report = createFailureReport(error, {
+        service: configuration.service,
+        context: {
+          environment: configuration.environment,
+          release: configuration.release,
+          region: configuration.region,
+          ...context
+        },
+        errorId: context.errorReference ?? `err_${generateId().replaceAll("-", "").toLowerCase()}`,
+        timestamp: now().toISOString(),
+        redactedValues: configuration.redactedValues
+      });
+    } catch {
+      report = createFailureReport(error, {
+        service: configuration.service,
+        errorId,
+        redactedValues: configuration.redactedValues,
+        context: { reportingFailure: true }
+      });
+    }
+    try {
+      const lines = encodeReport(report);
+      if (lines.length > 1) {
+        let budget = 2e3;
+        let summary;
+        do {
+          summary = JSON.stringify({
+            event: "diagnostics.summary",
+            errorId: report.errorId,
+            tree: renderFailure(report, budget),
+            completeReport: "Reconstruct diagnostics.chunk entries by errorId"
+          });
+          budget = Math.floor(budget / 2);
+        } while (Buffer.byteLength(summary) > 4096 && budget > 128);
+        if (Buffer.byteLength(summary) <= 4096) write(context.severity ?? "error", summary);
+      }
+      for (const line of lines) write(context.severity ?? "error", line);
+    } catch {
+      report.context.reportingFailure = true;
+      try {
+        for (const line of encodeReport(report)) process.stderr.write(line + "\n");
+      } catch {
+      }
+    }
+    try {
+      observe?.(report);
+    } catch {
+    }
+    return report;
+  }
+  return { capture, report: (error, context) => capture(error, context).errorId };
+}
+
+// ../diagnostics/src/runtime.js
+function captureRuntimeFailure(error, context, environment = process.env, values = []) {
+  const { service, ...details } = context;
+  return makeErrorReporter({
+    configuration: { service, redactedValues: [...environmentSecrets(environment), ...values] }
+  }).capture(error, details);
+}
+export {
+  captureRuntimeFailure,
+  environmentSecrets,
+  makeRedactor
+};