@effectstream/log 0.100.0

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@effectstream/log",
+  "version": "0.100.0",
+  "license": "MIT",
+  "exports": "./src/mod.ts",
+  "dependencies": {
+    "@effectstream/utils": "0.100.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/api-logs": "0.56.0",
+    "@opentelemetry/exporter-logs-otlp-http": "0.56.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "0.56.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.56.0",
+    "@opentelemetry/resources": "1.29.0",
+    "@opentelemetry/sdk-logs": "0.56.0",
+    "@opentelemetry/sdk-metrics": "1.29.0",
+    "@opentelemetry/sdk-trace-base": "1.29.0",
+    "@opentelemetry/semantic-conventions": "1.28.0",
+    "chalk": "5.4.1",
+    "effection": "3.5.0",
+    "material-chalk": "1.1.1",
+    "superjson": "^2.2.1",
+    "tslog": "^4.9.3"
+  }
+}

package/src/brands.ts

@@ -0,0 +1,10 @@
+import { Format, matchColor, registerBrand } from "material-chalk";
+
+const paimaBrandColor = matchColor("#1BB17B");
+registerBrand("paima", paimaBrandColor.formatAs(Format.Hct));
+
+const hardhatBrandColor = matchColor("#FFF100");
+registerBrand("hardhat", hardhatBrandColor.formatAs(Format.Hct));
+
+const dolosBrandColor = matchColor("#FF0065");
+registerBrand("dolos", dolosBrandColor.formatAs(Format.Hct));

package/src/const.ts

@@ -0,0 +1,108 @@
+import { SeverityNumber } from "@opentelemetry/api-logs";
+import { getEnv } from "@effectstream/utils/runtime";
+
+/**
+ * Main components
+ * These components get a "paima" prefix in the log.
+ */
+export const PaimaComponents = {
+  EFFECTSTREAM_SYNC: "sync",
+  EFFECTSTREAM_RUNTIME: "runtime",
+  EFFECTSTREAM_PGLITE: "pglite",
+  APPLY_MIGRATIONS: "apply-migrations",
+};
+/**
+ * External components
+ */
+export const ExternalComponents = {
+  HARDHAT: "hardhat",
+  YACI_DEVKIT: "yaci-devkit",
+  DOLOS: "dolos",
+  MIDNIGHT_NODE: "midnight-node",
+  MIDNIGHT_INDEXER: "midnight-indexer",
+  MIDNIGHT_PROOF_SERVER: "midnight-proof-server",
+  AVAIL_NODE: "avail-node",
+  AVAIL_CLIENT: "avail-light-client",
+  BITCOIN_CORE: "bitcoin-core",
+  NEAR_SANDBOX: "near-sandbox",
+};
+
+/**
+ * Paima Tools, helper and services to support the main components.
+ * These components get a "paima" prefix in the log.
+ */
+export const PaimaToolsComponents = {
+  CHECKER: "checker",
+  TUI: "tui",
+  TMUX: "tmux",
+  COLLECTOR: "collector",
+  EXPLORER: "explorer",
+  DEPLOY_EVM_CONTRACTS: "deploy-evm-contracts",
+  MIDNIGHT_CONTRACT: "midnight-contract",
+};
+/**
+ * Secondary components
+ * These components are launched automatically.
+ */
+export const SecondaryComponents = {
+  // This is the entry point, so it gets launched by default.
+  ORCHESTRATOR: "orchestrator",
+  // Thses processes are launched by their counterpart.
+  DOLOS_FILL_TEMPLATE: "dolos-fill-template",
+  DOLOS_WAIT: "dolos-wait",
+  YACI_DEVKIT_WAIT: "yaci-devkit-wait",
+  MIDNIGHT_NODE_WAIT: "midnight-node-wait",
+  MIDNIGHT_INDEXER_WAIT: "midnight-indexer-wait",
+  MIDNIGHT_PROOF_SERVER_WAIT: "midnight-proof-server-wait",
+  BITCOIN_CORE_WAIT: "bitcoin-core-wait",
+  BITCOIN_GENERATE_BLOCKS: "bitcoin-mine-blocks",
+  BITCOIN_WAIT_FOR_BLOCK: "bitcoin-wait-for-block",
+  AVAIL_NODE_WAIT: "avail-node-wait",
+  AVAIL_CLIENT_WAIT: "avail-light-client-wait",
+  NEAR_SANDBOX_WAIT: "near-sandbox-wait",
+  HARDHAT_WAIT: "hardhat-wait",
+  COLLECTOR_WAIT: "collector-wait",
+  EFFECTSTREAM_DB_WAIT: "db-wait",
+  LOKI: "loki",
+};
+
+/** All the components that can be launched by the orchestrator */
+export const LaunchableComponents = {
+  ...PaimaComponents,
+  ...PaimaToolsComponents,
+  ...ExternalComponents,
+};
+/** All the components */
+export const ComponentNames = {
+  ...LaunchableComponents,
+  ...SecondaryComponents,
+};
+
+const parseSeverity = (): SeverityNumber => {
+  const level = getEnv("EFFECTSTREAM_LOG_LEVEL")?.toUpperCase();
+  if (level && level in SeverityNumber) {
+    return SeverityNumber[level as keyof typeof SeverityNumber];
+  }
+  return SeverityNumber.INFO;
+};
+
+export const defaultSeverity = parseSeverity();
+
+/**
+ * extra step in the logger to avoid performance degradation.
+ * otherwise, the following can slow down the program:
+ * `log.local(LogLevels.DEBUG, JSON.stringify(largeObj))`
+ * since it runs `JSON.stringify` even if the default `LogLevel` excludes it
+ */
+export type DeferredLog = (logger: typeof console.log) => void;
+
+// TODO: should re-export a type built-into material-chalk instead of defining it here
+//       notably, https://github.com/SebastienGllmt/material-chalk/blob/master/src/core.ts#L175
+export type Namespace = string | string[];
+
+export type LogFunc = (
+  component: string,
+  namespace: Namespace,
+  level: SeverityNumber,
+  doLog: DeferredLog,
+) => void;

package/src/mod.ts

@@ -0,0 +1,35 @@
+import { SeverityNumber } from "@opentelemetry/api-logs";
+import { tsLogFormatted, tsLogGetFormattedMessage, tsLogOrchestrator, type TslogLogFunc } from "./tslog.ts";
+import { otelLog, type OtelLogFunc } from "./otel/logger.ts";
+import "./brands.ts"; // register material-chalk brands
+import { getEnv } from "@effectstream/utils/runtime";
+export * from "./otel/setup.ts";
+export {
+  ComponentNames,
+  LaunchableComponents,
+  type Namespace,
+} from "./const.ts";
+export { attachTransport } from "./tslog.ts";
+export { DefaultLogLevels } from "./tslog.ts";
+
+// re-exporting this
+// so that we don't need to re-import opentelemetry in every component
+export { SeverityNumber };
+
+const orchestratorEnv = getEnv("EFFECTSTREAM_ORCHESTRATOR");
+const localLogger = orchestratorEnv ? tsLogOrchestrator : tsLogFormatted;
+const remoteLogger = orchestratorEnv ? tsLogOrchestrator : otelLog;
+
+export const log: {
+  local: TslogLogFunc;
+  localForce: TslogLogFunc;
+  remote: TslogLogFunc;
+  remoteForce: OtelLogFunc;
+  formatMessage: typeof tsLogGetFormattedMessage
+} = {
+  local: localLogger,
+  remote: remoteLogger,
+  localForce: tsLogFormatted,
+  remoteForce: otelLog,
+  formatMessage: tsLogGetFormattedMessage,
+};

package/src/otel/logger.ts

@@ -0,0 +1,32 @@
+import * as otlApi from "@opentelemetry/api-logs";
+import type { LogFunc } from "../const.ts";
+import { defaultSeverity } from "../const.ts";
+import { toString } from "../utils.ts";
+
+export type OtelLogFunc = LogFunc;
+
+export const otelLog: OtelLogFunc = (
+  component,
+  namespace,
+  level,
+  doLog,
+): void => {
+  if (level < defaultSeverity) return;
+  // note: we can't just use otel span events, or debugging gets really hard
+  //       since span events only get sent to the collector after the span completed.
+  //       there are some open questions about logs/span/events too, so we just don't use span events for now
+  //       See https://github.com/open-telemetry/opentelemetry-specification/discussions/3732
+
+  // TODO: modify these constants
+  const logger = otlApi.logs.getLogger(component, "0.3.0");
+  doLog((...data: unknown[]) => {
+    return logger.emit({
+      severityNumber: level,
+      body: data.map(toString).join(" "),
+      attributes: {
+        component,
+        namespace: Array.isArray(namespace) ? namespace.join(".") : namespace,
+      },
+    })
+  });
+};

package/src/otel/setup.ts

@@ -0,0 +1,104 @@
+// careful: only include opentelemetry libs that work in the browser too
+import type opentelemetry from "@opentelemetry/api";
+import { createContext, type Operation } from "effection";
+import { type IResource, Resource } from "@opentelemetry/resources";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
+import {
+  BatchLogRecordProcessor,
+  type LogRecordProcessor,
+  SimpleLogRecordProcessor,
+} from "@opentelemetry/sdk-logs";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import type { MetricReader } from "@opentelemetry/sdk-metrics";
+import {
+  BatchSpanProcessor,
+  SimpleSpanProcessor,
+  type SpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import {
+  ATTR_SERVICE_NAME,
+  ATTR_SERVICE_VERSION,
+} from "@opentelemetry/semantic-conventions";
+import { env } from "node:process";
+
+/**
+ * Note: this is based on `NodeSDKConfiguration`.
+ *       We just don't want to include a nodejs dependency here
+ */
+type DefaultOtelSetup = {
+  resource: IResource;
+  logRecordProcessors: LogRecordProcessor[];
+  metricReader: MetricReader;
+  spanProcessors?: SpanProcessor[];
+};
+export function defaultOtelSetup(
+  name: string,
+  version: string,
+): DefaultOtelSetup {
+  const collectorOptions = {
+    // override default on dev builds so no logs disappear
+    concurrencyLimit: env.NODE_ENV === "development"
+      ? Number.MAX_SAFE_INTEGER
+      : undefined,
+  } as const;
+
+  const baseLog = new OTLPLogExporter(collectorOptions);
+  const logProcessor = env.NODE_ENV === "development"
+    ? new SimpleLogRecordProcessor(baseLog)
+    : new BatchLogRecordProcessor(baseLog);
+
+  const baseTrace = new OTLPTraceExporter(collectorOptions);
+  const spanProcessor = env.NODE_ENV === "development"
+    ? new SimpleSpanProcessor(baseTrace)
+    : new BatchSpanProcessor(baseTrace);
+
+  return {
+    resource: Resource.default().merge(
+      new Resource({
+        [ATTR_SERVICE_NAME]: name,
+        [ATTR_SERVICE_VERSION]: version,
+      }),
+    ),
+    spanProcessors: [spanProcessor],
+    logRecordProcessors: [logProcessor],
+    metricReader: new PeriodicExportingMetricReader({
+      exporter: new OTLPMetricExporter(collectorOptions),
+    }),
+  };
+}
+
+type TelemetryContext = {
+  tracer: opentelemetry.Tracer;
+  span: undefined | opentelemetry.Span;
+};
+export const PaimaTelemetryContext = createContext<TelemetryContext>(
+  "paima:telemetry",
+);
+
+export function* useTelemetryContext(): Operation<TelemetryContext> {
+  const otel = yield* PaimaTelemetryContext.get();
+  if (!otel) {
+    throw new Error("Paima telemetry is not initialized");
+  }
+  return otel;
+}
+export function* withSpan<T>(
+  name: string,
+  continuation: () => Operation<T>,
+): Operation<T> {
+  const otel = yield* useTelemetryContext();
+  return yield* otel.tracer.startActiveSpan(
+    name,
+    function* (span: opentelemetry.Span) {
+      yield* PaimaTelemetryContext.set(
+        {
+          tracer: otel.tracer,
+          span,
+        },
+      );
+      return yield* continuation();
+    },
+  );
+}

package/src/tslog.ts

@@ -0,0 +1,194 @@
+import { SeverityNumber } from "@opentelemetry/api-logs";
+import { type ILogObj, type IMeta, Logger } from "tslog";
+import {
+  defaultSeverity,
+  type LogFunc,
+  type Namespace,
+  PaimaComponents,
+  PaimaToolsComponents,
+} from "./const.ts";
+import { chainedMessage, Format, matchColor } from "material-chalk";
+import chalk from "chalk";
+import { toString } from "./utils.ts";
+// TODO How to handle this in the frontend & backend?
+const ENV = { NODE_ENV: 'development' } // TODO This should come from @effectstream/utils/node-env ENV
+
+/**
+ * https://github.com/fullstack-build/tslog/pull/308
+ */
+export enum DefaultLogLevels {
+  SILLY = 0,
+  TRACE = 1,
+  DEBUG = 2,
+  INFO = 3,
+  WARN = 4,
+  ERROR = 5,
+  FATAL = 6,
+}
+
+export function mapSeverity(level: SeverityNumber): DefaultLogLevels {
+  if (level === SeverityNumber.UNSPECIFIED) {
+    return DefaultLogLevels.INFO;
+  }
+  if (level < SeverityNumber.DEBUG) {
+    return DefaultLogLevels.TRACE;
+  }
+  if (level < SeverityNumber.INFO) {
+    return DefaultLogLevels.DEBUG;
+  }
+  if (level < SeverityNumber.WARN) {
+    return DefaultLogLevels.INFO;
+  }
+  if (level < SeverityNumber.ERROR) {
+    return DefaultLogLevels.WARN;
+  }
+  if (level < SeverityNumber.FATAL) {
+    return DefaultLogLevels.ERROR;
+  }
+  return DefaultLogLevels.FATAL;
+}
+
+const chalkFormat = Format.Chalk(chalk);
+const logLevels = {
+  SILLY: matchColor("#FFFFFF").formatAs(chalkFormat).dim,
+  TRACE: matchColor("#FFFFFF").formatAs(chalkFormat).dim,
+  DEBUG: matchColor("#00FF00").formatAs(chalkFormat).dim,
+  INFO: matchColor("#0000FF").formatAs(chalkFormat).dim,
+  WARN: matchColor("#FFFF00").formatAs(chalkFormat),
+  ERROR: matchColor("#FF0000").formatAs(chalkFormat),
+  FATAL: matchColor("#FF0000").formatAs(chalkFormat),
+};
+
+function padLeft(
+  input: number | string,
+  targetLength: number,
+  padChar: string = "0",
+): string {
+  const str = input.toString();
+  return str.padStart(targetLength, padChar);
+}
+
+const log: Logger<ILogObj> = new Logger({
+  minLevel: mapSeverity(defaultSeverity),
+  overwrite: {
+    formatMeta: (meta?: IMeta) => {
+      if (meta == null) return "";
+      const timestamp = chalk.white.dim((() => {
+        if (ENV.NODE_ENV !== "development") {
+          return meta.date.toISOString();
+        }
+        // use a shorter timestamp locally so it takes less space on the console
+        return `${padLeft(meta.date.getHours(), 2)}:${
+          padLeft(meta.date.getMinutes(), 2)
+        }:${padLeft(meta.date.getSeconds(), 2)}`;
+      })());
+      const logLevelName = logLevels
+        [meta.logLevelName as keyof typeof logLevels](meta.logLevelName);
+      return [timestamp, logLevelName].join(" ") + "\t";
+    },
+  },
+});
+
+export function attachTransport(callback: (logObj: ILogObj) => void) {
+  log.attachTransport((logObj) => {
+    callback(logObj);
+  });
+}
+
+function getLogMethod(level: SeverityNumber) {
+  if (level === SeverityNumber.UNSPECIFIED) {
+    const level = mapSeverity(defaultSeverity);
+    return log.log.bind(
+      log,
+      level,
+      DefaultLogLevels[level],
+    );
+  }
+  if (level < SeverityNumber.DEBUG) {
+    return log.trace.bind(log);
+  }
+  if (level < SeverityNumber.INFO) {
+    return log.debug.bind(log);
+  }
+  if (level < SeverityNumber.WARN) {
+    return log.info.bind(log);
+  }
+  if (level < SeverityNumber.ERROR) {
+    return log.warn.bind(log);
+  }
+  if (level < SeverityNumber.FATAL) {
+    return log.error.bind(log);
+  }
+  return log.fatal.bind(log);
+}
+
+export type TslogLogFunc = LogFunc;
+
+const formatMessage = (
+  namespaces: Namespace,
+  ...data: unknown[]
+): string => {
+  return chainedMessage(
+    chalk,
+    namespaces,
+    ...data.map(toString),
+  );
+};
+
+
+const namespaces = (component: string, namespace: Namespace) => {
+  const subNamespaces = Array.isArray(namespace) ? namespace : [namespace];
+  if (
+    Object.values(PaimaComponents).includes(component as any) ||
+    Object.values(PaimaToolsComponents).includes(component as any)
+  ) {
+    return ["effectstream", component, ...subNamespaces];
+  }
+  return [component, ...subNamespaces];
+};
+
+export const tsLogFormatted: TslogLogFunc = (
+  component,
+  namespace,
+  level,
+  doLog,
+): void => {
+  doLog((...data: unknown[]) =>
+    getLogMethod(level)(
+      formatMessage(namespaces(component, namespace), ...data),
+    )
+  );
+};
+
+export const tsLogOrchestrator: TslogLogFunc = (
+  component,
+  namespace,
+  level,
+  doLog,
+): void => {
+  if (level < defaultSeverity) return;
+  doLog((...data: unknown[]) =>
+      console.log(JSON.stringify({
+        "__ORCHESTRATOR__": true,
+        data,
+        namespaces: namespaces(component, namespace),
+        level,
+        date: new Date().getTime(),
+      }, 
+      (_, v) => typeof v === "bigint" ? v.toString() : v
+    ))
+  )
+};
+
+export const tsLogGetFormattedMessage = (
+  component: string,
+  namespace: Namespace,
+  level: SeverityNumber,
+  doLog: any,
+): string => {
+  let val = "";
+  doLog((...data: unknown[]) => {
+    val = formatMessage(namespaces(component, namespace), ...data);
+  });
+  return val;
+};

package/src/utils.ts

@@ -0,0 +1,20 @@
+import superjson from "superjson";
+
+export function toString(str: unknown): string {
+  if (typeof str !== "object") {
+    if (typeof str === "function") {
+      return str.name === "" ? "[Function]" : `[Function: ${str.name}]`;
+    }
+    return (String(str));
+  }
+  if (str instanceof Error) {
+    return (`${str.name}: ${str.message}\nStack: ${str.stack}`);
+  }
+
+  try {
+    return superjson.stringify(str);
+  } catch (e) {
+    return toString(e);
+    // should not happen, but maybe there is some type that fails for some reason
+  }
+}