@rocicorp/zero 1.3.0 → 1.4.0-canary.0

package/out/zero-cache/src/server/anonymous-otel-start.js.map

@@ -1 +1 @@
-{"version":3,"file":"anonymous-otel-start.js","names":["#instance","#processId","#lc","#starting","#config","#viewSyncerCount","#cachedAttributes","#run","#stopped","#getAttributes","#meterProvider","#meter","#setupMetrics","#activeUsersGetter","#totalCrudMutations","#totalCustomMutations","#totalCrudQueries","#totalCustomQueries","#totalRowsSynced","#totalConnectionsSuccess","#totalConnectionsAttempted","#activeClientGroupsGetter","#getPlatform","#getGitProjectId","#getOrSetFsID","#findUp","#isInContainer"],"sources":["../../../../../zero-cache/src/server/anonymous-otel-start.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport {type Meter} from '@opentelemetry/api';\nimport {OTLPMetricExporter} from '@opentelemetry/exporter-metrics-otlp-http';\nimport {resourceFromAttributes} from '@opentelemetry/resources';\nimport {\n  MeterProvider,\n  PeriodicExportingMetricReader,\n} from '@opentelemetry/sdk-metrics';\nimport type {LogContext} from '@rocicorp/logger';\nimport {execSync} from 'child_process';\nimport {randomUUID} from 'crypto';\nimport {existsSync, mkdirSync, readFileSync, writeFileSync} from 'fs';\nimport {homedir, platform} from 'os';\nimport {dirname, join} from 'path';\nimport {h64} from '../../../shared/src/hash.js';\nimport {\n  getServerVersion,\n  getZeroConfig,\n  type ZeroConfig,\n} from '../config/zero-config.js';\nimport {setupOtelDiagnosticLogger} from './otel-diag-logger.ts';\n\nexport type ActiveUsers = {\n  active_users_last_day: number;\n  users_1da: number;\n  users_7da: number;\n  users_30da: number;\n  users_1da_legacy: number;\n  users_7da_legacy: number;\n  users_30da_legacy: number;\n};\n\nconst hostNameRe = /^[a-f0-9]{12}$/;\n\nclass AnonymousTelemetryManager {\n  static #instance: AnonymousTelemetryManager;\n  #starting = false;\n  #stopped = false;\n  #meter!: Meter;\n  #meterProvider!: MeterProvider;\n  #totalCrudMutations = 0;\n  #totalCustomMutations = 0;\n  #totalCrudQueries = 0;\n  #totalCustomQueries = 0;\n  #totalRowsSynced = 0;\n  #totalConnectionsSuccess = 0;\n  #totalConnectionsAttempted = 0;\n  #activeClientGroupsGetter: (() => number) | undefined;\n  #activeUsersGetter: (() => ActiveUsers) | undefined;\n  #lc: LogContext | undefined;\n  #config: ZeroConfig | undefined;\n  #processId: string;\n  #cachedAttributes: Record<string, string> | undefined;\n  #viewSyncerCount = 1;\n\n  private constructor() {\n    this.#processId = randomUUID();\n  }\n\n  static getInstance(): AnonymousTelemetryManager {\n    if (!AnonymousTelemetryManager.#instance) {\n      AnonymousTelemetryManager.#instance = new AnonymousTelemetryManager();\n    }\n    return AnonymousTelemetryManager.#instance;\n  }\n\n  start(lc?: LogContext, config?: ZeroConfig) {\n    this.#lc = lc;\n\n    // Set up OpenTelemetry diagnostic logger if not already configured\n    setupOtelDiagnosticLogger(lc);\n\n    if (!config) {\n      try {\n        config = getZeroConfig();\n      } catch (e) {\n        this.#lc?.info?.('telemetry: disabled - unable to parse config', e);\n        return;\n      }\n    }\n\n    if (process.env.DO_NOT_TRACK) {\n      this.#lc?.info?.(\n        'telemetry: disabled - DO_NOT_TRACK environment variable is set',\n      );\n      return;\n    }\n\n    if (!config.enableTelemetry) {\n      this.#lc?.info?.('telemetry: disabled - enableTelemetry is false');\n      return;\n    }\n\n    if (this.#starting) {\n      return;\n    }\n\n    this.#starting = true;\n    this.#config = config;\n    this.#viewSyncerCount = config.numSyncWorkers ?? 1;\n    this.#cachedAttributes = undefined;\n\n    this.#lc?.info?.(`telemetry: starting in 1 minute`);\n\n    // Delay telemetry startup by 1 minute to avoid potential boot loop issues\n    setTimeout(() => this.#run(), 60000);\n  }\n\n  #run() {\n    if (this.#stopped) {\n      return;\n    }\n\n    const resource = resourceFromAttributes(this.#getAttributes());\n\n    // Add a random jitter to the export interval to avoid all view-syncers exporting at the same time\n    const exportIntervalMillis =\n      60000 * this.#viewSyncerCount + Math.floor(Math.random() * 10000);\n    const readers = [\n      new PeriodicExportingMetricReader({\n        exportIntervalMillis,\n        exporter: new OTLPMetricExporter({\n          url: 'https://metrics.rocicorp.dev',\n          timeoutMillis: 30000,\n        }),\n      }),\n    ];\n\n    // Uncomment this to debug metrics exports.\n\n    // readers.push(\n    //   new PeriodicExportingMetricReader({\n    //     exportIntervalMillis,\n    //     exporter: new ConsoleMetricExporter(),\n    //   }),\n    // );\n\n    this.#meterProvider = new MeterProvider({resource, readers});\n    this.#meter = this.#meterProvider.getMeter('zero-anonymous-telemetry');\n\n    this.#setupMetrics();\n    this.#lc?.info?.(\n      `telemetry: started (exports every ${exportIntervalMillis / 1000} seconds for ${this.#viewSyncerCount} view-syncers)`,\n    );\n  }\n\n  #setupMetrics() {\n    // Observable gauges\n    const uptimeGauge = this.#meter.createObservableGauge('zero.uptime', {\n      description: 'System uptime in seconds',\n      unit: 'seconds',\n    });\n\n    // Observable counters\n    const uptimeCounter = this.#meter.createObservableCounter(\n      'zero.uptime_counter',\n      {\n        description: 'System uptime in seconds',\n        unit: 'seconds',\n      },\n    );\n    const crudMutationsCounter = this.#meter.createObservableCounter(\n      'zero.crud_mutations_processed',\n      {\n        description: 'Total number of CRUD mutations processed',\n      },\n    );\n    const customMutationsCounter = this.#meter.createObservableCounter(\n      'zero.custom_mutations_processed',\n      {\n        description: 'Total number of custom mutations processed',\n      },\n    );\n    const totalMutationsCounter = this.#meter.createObservableCounter(\n      'zero.mutations_processed',\n      {\n        description: 'Total number of mutations processed',\n      },\n    );\n    const crudQueriesCounter = this.#meter.createObservableCounter(\n      'zero.crud_queries_processed',\n      {\n        description: 'Total number of CRUD queries processed',\n      },\n    );\n    const customQueriesCounter = this.#meter.createObservableCounter(\n      'zero.custom_queries_processed',\n      {\n        description: 'Total number of custom queries processed',\n      },\n    );\n    const totalQueriesCounter = this.#meter.createObservableCounter(\n      'zero.queries_processed',\n      {\n        description: 'Total number of queries processed',\n      },\n    );\n    const rowsSyncedCounter = this.#meter.createObservableCounter(\n      'zero.rows_synced',\n      {\n        description: 'Total number of rows synced',\n      },\n    );\n\n    // Observable counters for connections\n    const connectionsSuccessCounter = this.#meter.createObservableCounter(\n      'zero.connections_success',\n      {\n        description: 'Total number of successful connections',\n      },\n    );\n\n    const connectionsAttemptedCounter = this.#meter.createObservableCounter(\n      'zero.connections_attempted',\n      {\n        description: 'Total number of attempted connections',\n      },\n    );\n\n    const activeClientGroupsGauge = this.#meter.createObservableGauge(\n      'zero.gauge_active_client_groups',\n      {\n        description: 'Number of currently active client groups',\n      },\n    );\n\n    const attrs = this.#getAttributes();\n    const active =\n      (metric: keyof ActiveUsers) => (result: ObservableResult) => {\n        const actives = this.#activeUsersGetter?.();\n        if (actives) {\n          const value = actives[metric];\n          result.observe(value, attrs);\n        }\n      };\n    this.#meter\n      .createObservableGauge('zero.active_users_last_day', {\n        description: 'Count of CVR instances active in the last 24h',\n      })\n      .addCallback(active('active_users_last_day'));\n    this.#meter\n      .createObservableGauge('zero.users_1da', {\n        description: 'Count of 1-day active profiles',\n      })\n      .addCallback(active('users_1da'));\n    this.#meter\n      .createObservableGauge('zero.users_7da', {\n        description: 'Count of 7-day active profiles',\n      })\n      .addCallback(active('users_7da'));\n    this.#meter\n      .createObservableGauge('zero.users_30da', {\n        description: 'Count of 30-day active profiles',\n      })\n      .addCallback(active('users_30da'));\n    this.#meter\n      .createObservableGauge('zero.users_1da_legacy', {\n        description: 'Count of 1-day active profiles with CVR fallback',\n      })\n      .addCallback(active('users_1da_legacy'));\n    this.#meter\n      .createObservableGauge('zero.users_7da_legacy', {\n        description: 'Count of 7-day active profiles with CVR fallback',\n      })\n      .addCallback(active('users_7da_legacy'));\n    this.#meter\n      .createObservableGauge('zero.users_30da_legacy', {\n        description: 'Count of 30-day active profiles with CVR fallback',\n      })\n      .addCallback(active('users_30da_legacy'));\n\n    // Callbacks\n    uptimeGauge.addCallback((result: ObservableResult) => {\n      const uptimeSeconds = Math.floor(process.uptime());\n      result.observe(uptimeSeconds, attrs);\n    });\n    uptimeCounter.addCallback((result: ObservableResult) => {\n      const uptimeSeconds = Math.floor(process.uptime());\n      result.observe(uptimeSeconds, attrs);\n    });\n    crudMutationsCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCrudMutations, attrs);\n    });\n    customMutationsCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCustomMutations, attrs);\n    });\n    totalMutationsCounter.addCallback((result: ObservableResult) => {\n      const totalMutations =\n        this.#totalCrudMutations + this.#totalCustomMutations;\n      result.observe(totalMutations, attrs);\n    });\n    crudQueriesCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCrudQueries, attrs);\n    });\n    customQueriesCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCustomQueries, attrs);\n    });\n    totalQueriesCounter.addCallback((result: ObservableResult) => {\n      const totalQueries = this.#totalCrudQueries + this.#totalCustomQueries;\n      result.observe(totalQueries, attrs);\n    });\n    rowsSyncedCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalRowsSynced, attrs);\n    });\n    connectionsSuccessCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalConnectionsSuccess, attrs);\n    });\n    connectionsAttemptedCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalConnectionsAttempted, attrs);\n    });\n    activeClientGroupsGauge.addCallback((result: ObservableResult) => {\n      const activeClientGroups = this.#activeClientGroupsGetter?.() ?? 0;\n      result.observe(activeClientGroups, attrs);\n    });\n  }\n\n  recordMutation(type: 'crud' | 'custom', count = 1) {\n    if (type === 'crud') {\n      this.#totalCrudMutations += count;\n    } else {\n      this.#totalCustomMutations += count;\n    }\n  }\n\n  recordQuery(type: 'crud' | 'custom', count = 1) {\n    if (type === 'crud') {\n      this.#totalCrudQueries += count;\n    } else {\n      this.#totalCustomQueries += count;\n    }\n  }\n\n  recordRowsSynced(count: number) {\n    this.#totalRowsSynced += count;\n  }\n\n  recordConnectionSuccess() {\n    this.#totalConnectionsSuccess++;\n  }\n\n  recordConnectionAttempted() {\n    this.#totalConnectionsAttempted++;\n  }\n\n  setActiveClientGroupsGetter(getter: () => number) {\n    this.#activeClientGroupsGetter = getter;\n  }\n\n  setActiveUsersGetter(getter: () => ActiveUsers) {\n    this.#activeUsersGetter = getter;\n  }\n\n  shutdown() {\n    this.#stopped = true;\n    if (this.#meterProvider) {\n      this.#lc?.info?.('telemetry: shutting down');\n      void this.#meterProvider.shutdown();\n    }\n  }\n\n  #getAttributes() {\n    if (!this.#cachedAttributes) {\n      this.#cachedAttributes = {\n        'zero.app.id': h64(this.#config?.upstream.db || 'unknown').toString(),\n        'zero.machine.os': platform(),\n        'zero.telemetry.type': 'anonymous',\n        'zero.infra.platform': this.#getPlatform(),\n        'zero.version': getServerVersion(this.#config),\n        'zero.task.id': this.#config?.taskID || 'unknown',\n        'zero.project.id': this.#getGitProjectId(),\n        'zero.process.id': this.#processId,\n        'zero.fs.id': this.#getOrSetFsID(),\n      };\n      this.#lc?.debug?.(\n        `telemetry: cached attributes=${JSON.stringify(this.#cachedAttributes)}`,\n      );\n    }\n    return this.#cachedAttributes;\n  }\n\n  #getPlatform(): string {\n    if (process.env.ZERO_ON_CLOUD_ZERO) return 'cloudzero';\n    if (process.env.FLY_APP_NAME || process.env.FLY_REGION) return 'fly.io';\n    if (\n      process.env.ECS_CONTAINER_METADATA_URI_V4 ||\n      process.env.ECS_CONTAINER_METADATA_URI ||\n      process.env.AWS_EXECUTION_ENV\n    ) {\n      return 'aws';\n    }\n    if (process.env.RAILWAY_ENV || process.env.RAILWAY_STATIC_URL) {\n      return 'railway';\n    }\n    if (process.env.RENDER || process.env.RENDER_SERVICE_ID) return 'render';\n    if (\n      process.env.GCP_PROJECT ||\n      process.env.GCLOUD_PROJECT ||\n      process.env.GOOGLE_CLOUD_PROJECT\n    ) {\n      return 'gcp';\n    }\n    if (process.env.COOLIFY_URL || process.env.COOLIFY_CONTAINER_NAME) {\n      return 'coolify';\n    }\n    if (process.env.CONTAINER_APP_REVISION) return 'azure';\n    if (process.env.FLIGHTCONTROL || process.env.FC_URL) return 'flightcontrol';\n    return 'unknown';\n  }\n\n  #findUp(startDir: string, target: string): string | null {\n    let dir = startDir;\n    while (dir !== dirname(dir)) {\n      if (existsSync(join(dir, target))) return dir;\n      dir = dirname(dir);\n    }\n    return null;\n  }\n\n  #getGitProjectId(): string {\n    try {\n      const cwd = process.cwd();\n      const gitRoot = this.#findUp(cwd, '.git');\n      if (!gitRoot) {\n        return 'unknown';\n      }\n\n      const rootCommitHash = execSync('git rev-list --max-parents=0 HEAD -1', {\n        cwd: gitRoot,\n        encoding: 'utf8',\n        timeout: 1000,\n        stdio: ['ignore', 'pipe', 'ignore'], // Suppress stderr\n      }).trim();\n\n      return rootCommitHash.length === 40 ? rootCommitHash : 'unknown';\n    } catch (error) {\n      // execSync throws a child_process.SpawnSyncReturns-shaped error with an\n      // output property (an array with stdio buffers) and sometimes a killed\n      // property that references back to the error itself — making it\n      // circular and causing stringify() in the Logger logic to throw.\n      const details =\n        error instanceof Error\n          ? {message: error.message, name: error.name, stack: error.stack}\n          : String(error);\n      this.#lc?.debug?.('telemetry: unable to get Git root commit:', details);\n      return 'unknown';\n    }\n  }\n\n  #getOrSetFsID(): string {\n    try {\n      if (this.#isInContainer()) {\n        return 'container';\n      }\n      const fsidPath = join(homedir(), '.rocicorp', 'fsid');\n      const fsidDir = dirname(fsidPath);\n\n      mkdirSync(fsidDir, {recursive: true});\n\n      // Always try atomic file creation first - this eliminates any race conditions\n      const newId = randomUUID();\n      try {\n        writeFileSync(fsidPath, newId, {encoding: 'utf8', flag: 'wx'});\n        return newId;\n      } catch (writeError) {\n        if ((writeError as NodeJS.ErrnoException).code === 'EEXIST') {\n          const existingId = readFileSync(fsidPath, 'utf8').trim();\n          return existingId;\n        }\n        throw writeError;\n      }\n    } catch (error) {\n      this.#lc?.debug?.(\n        'telemetry: unable to get or set filesystem ID:',\n        error,\n      );\n      return 'unknown';\n    }\n  }\n\n  #isInContainer(): boolean {\n    try {\n      if (process.env.ZERO_IN_CONTAINER) {\n        return true;\n      }\n\n      if (existsSync('/.dockerenv')) {\n        return true;\n      }\n\n      if (existsSync('/usr/local/bin/docker-entrypoint.sh')) {\n        return true;\n      }\n\n      if (process.env.KUBERNETES_SERVICE_HOST) {\n        return true;\n      }\n\n      if (\n        process.env.DOCKER_CONTAINER_ID ||\n        process.env.HOSTNAME?.match(hostNameRe)\n      ) {\n        return true;\n      }\n\n      if (existsSync('/proc/1/cgroup')) {\n        const cgroup = readFileSync('/proc/1/cgroup', 'utf8');\n        if (\n          cgroup.includes('docker') ||\n          cgroup.includes('kubepods') ||\n          cgroup.includes('containerd')\n        ) {\n          return true;\n        }\n      }\n\n      return false;\n    } catch (error) {\n      this.#lc?.debug?.(\n        'telemetry: unable to detect container environment:',\n        error,\n      );\n      return false;\n    }\n  }\n}\n\nconst manager = () => AnonymousTelemetryManager.getInstance();\n\nexport const startAnonymousTelemetry = (lc?: LogContext, config?: ZeroConfig) =>\n  manager().start(lc, config);\nexport const recordMutation = (type: 'crud' | 'custom', count = 1) =>\n  manager().recordMutation(type, count);\nexport const recordQuery = (type: 'crud' | 'custom', count = 1) =>\n  manager().recordQuery(type, count);\nexport const recordRowsSynced = (count: number) =>\n  manager().recordRowsSynced(count);\nexport const recordConnectionSuccess = () =>\n  manager().recordConnectionSuccess();\nexport const recordConnectionAttempted = () =>\n  manager().recordConnectionAttempted();\nexport const setActiveClientGroupsGetter = (getter: () => number) =>\n  manager().setActiveClientGroupsGetter(getter);\nexport const setActiveUsersGetter = (getter: () => ActiveUsers) =>\n  manager().setActiveUsersGetter(getter);\nexport const shutdownAnonymousTelemetry = () => manager().shutdown();\n"],"mappings":";;;;;;;;;;;;;AAgCA,IAAM,aAAa;AAEnB,IAAM,4BAAN,MAAM,0BAA0B;CAC9B,QAAA;CACA,YAAY;CACZ,WAAW;CACX;CACA;CACA,sBAAsB;CACtB,wBAAwB;CACxB,oBAAoB;CACpB,sBAAsB;CACtB,mBAAmB;CACnB,2BAA2B;CAC3B,6BAA6B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CAEnB,cAAsB;AACpB,QAAA,YAAkB,YAAY;;CAGhC,OAAO,cAAyC;AAC9C,MAAI,CAAC,2BAAA,SACH,4BAAA,WAAsC,IAAI,2BAA2B;AAEvE,SAAO,2BAAA;;CAGT,MAAM,IAAiB,QAAqB;AAC1C,QAAA,KAAW;AAGX,4BAA0B,GAAG;AAE7B,MAAI,CAAC,OACH,KAAI;AACF,YAAS,eAAe;WACjB,GAAG;AACV,SAAA,IAAU,OAAO,gDAAgD,EAAE;AACnE;;AAIJ,MAAI,QAAQ,IAAI,cAAc;AAC5B,SAAA,IAAU,OACR,iEACD;AACD;;AAGF,MAAI,CAAC,OAAO,iBAAiB;AAC3B,SAAA,IAAU,OAAO,iDAAiD;AAClE;;AAGF,MAAI,MAAA,SACF;AAGF,QAAA,WAAiB;AACjB,QAAA,SAAe;AACf,QAAA,kBAAwB,OAAO,kBAAkB;AACjD,QAAA,mBAAyB,KAAA;AAEzB,QAAA,IAAU,OAAO,kCAAkC;AAGnD,mBAAiB,MAAA,KAAW,EAAE,IAAM;;CAGtC,OAAO;AACL,MAAI,MAAA,QACF;EAGF,MAAM,WAAW,uBAAuB,MAAA,eAAqB,CAAC;EAG9D,MAAM,uBACJ,MAAQ,MAAA,kBAAwB,KAAK,MAAM,KAAK,QAAQ,GAAG,IAAM;AAoBnE,QAAA,gBAAsB,IAAI,cAAc;GAAC;GAAU,SAnBnC,CACd,IAAI,8BAA8B;IAChC;IACA,UAAU,IAAI,mBAAmB;KAC/B,KAAK;KACL,eAAe;KAChB,CAAC;IACH,CAAC,CACH;GAW0D,CAAC;AAC5D,QAAA,QAAc,MAAA,cAAoB,SAAS,2BAA2B;AAEtE,QAAA,cAAoB;AACpB,QAAA,IAAU,OACR,qCAAqC,uBAAuB,IAAK,eAAe,MAAA,gBAAsB,gBACvG;;CAGH,gBAAgB;EAEd,MAAM,cAAc,MAAA,MAAY,sBAAsB,eAAe;GACnE,aAAa;GACb,MAAM;GACP,CAAC;EAGF,MAAM,gBAAgB,MAAA,MAAY,wBAChC,uBACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EACD,MAAM,uBAAuB,MAAA,MAAY,wBACvC,iCACA,EACE,aAAa,4CACd,CACF;EACD,MAAM,yBAAyB,MAAA,MAAY,wBACzC,mCACA,EACE,aAAa,8CACd,CACF;EACD,MAAM,wBAAwB,MAAA,MAAY,wBACxC,4BACA,EACE,aAAa,uCACd,CACF;EACD,MAAM,qBAAqB,MAAA,MAAY,wBACrC,+BACA,EACE,aAAa,0CACd,CACF;EACD,MAAM,uBAAuB,MAAA,MAAY,wBACvC,iCACA,EACE,aAAa,4CACd,CACF;EACD,MAAM,sBAAsB,MAAA,MAAY,wBACtC,0BACA,EACE,aAAa,qCACd,CACF;EACD,MAAM,oBAAoB,MAAA,MAAY,wBACpC,oBACA,EACE,aAAa,+BACd,CACF;EAGD,MAAM,4BAA4B,MAAA,MAAY,wBAC5C,4BACA,EACE,aAAa,0CACd,CACF;EAED,MAAM,8BAA8B,MAAA,MAAY,wBAC9C,8BACA,EACE,aAAa,yCACd,CACF;EAED,MAAM,0BAA0B,MAAA,MAAY,sBAC1C,mCACA,EACE,aAAa,4CACd,CACF;EAED,MAAM,QAAQ,MAAA,eAAqB;EACnC,MAAM,UACH,YAA+B,WAA6B;GAC3D,MAAM,UAAU,MAAA,qBAA2B;AAC3C,OAAI,SAAS;IACX,MAAM,QAAQ,QAAQ;AACtB,WAAO,QAAQ,OAAO,MAAM;;;AAGlC,QAAA,MACG,sBAAsB,8BAA8B,EACnD,aAAa,iDACd,CAAC,CACD,YAAY,OAAO,wBAAwB,CAAC;AAC/C,QAAA,MACG,sBAAsB,kBAAkB,EACvC,aAAa,kCACd,CAAC,CACD,YAAY,OAAO,YAAY,CAAC;AACnC,QAAA,MACG,sBAAsB,kBAAkB,EACvC,aAAa,kCACd,CAAC,CACD,YAAY,OAAO,YAAY,CAAC;AACnC,QAAA,MACG,sBAAsB,mBAAmB,EACxC,aAAa,mCACd,CAAC,CACD,YAAY,OAAO,aAAa,CAAC;AACpC,QAAA,MACG,sBAAsB,yBAAyB,EAC9C,aAAa,oDACd,CAAC,CACD,YAAY,OAAO,mBAAmB,CAAC;AAC1C,QAAA,MACG,sBAAsB,yBAAyB,EAC9C,aAAa,oDACd,CAAC,CACD,YAAY,OAAO,mBAAmB,CAAC;AAC1C,QAAA,MACG,sBAAsB,0BAA0B,EAC/C,aAAa,qDACd,CAAC,CACD,YAAY,OAAO,oBAAoB,CAAC;AAG3C,cAAY,aAAa,WAA6B;GACpD,MAAM,gBAAgB,KAAK,MAAM,QAAQ,QAAQ,CAAC;AAClD,UAAO,QAAQ,eAAe,MAAM;IACpC;AACF,gBAAc,aAAa,WAA6B;GACtD,MAAM,gBAAgB,KAAK,MAAM,QAAQ,QAAQ,CAAC;AAClD,UAAO,QAAQ,eAAe,MAAM;IACpC;AACF,uBAAqB,aAAa,WAA6B;AAC7D,UAAO,QAAQ,MAAA,oBAA0B,MAAM;IAC/C;AACF,yBAAuB,aAAa,WAA6B;AAC/D,UAAO,QAAQ,MAAA,sBAA4B,MAAM;IACjD;AACF,wBAAsB,aAAa,WAA6B;GAC9D,MAAM,iBACJ,MAAA,qBAA2B,MAAA;AAC7B,UAAO,QAAQ,gBAAgB,MAAM;IACrC;AACF,qBAAmB,aAAa,WAA6B;AAC3D,UAAO,QAAQ,MAAA,kBAAwB,MAAM;IAC7C;AACF,uBAAqB,aAAa,WAA6B;AAC7D,UAAO,QAAQ,MAAA,oBAA0B,MAAM;IAC/C;AACF,sBAAoB,aAAa,WAA6B;GAC5D,MAAM,eAAe,MAAA,mBAAyB,MAAA;AAC9C,UAAO,QAAQ,cAAc,MAAM;IACnC;AACF,oBAAkB,aAAa,WAA6B;AAC1D,UAAO,QAAQ,MAAA,iBAAuB,MAAM;IAC5C;AACF,4BAA0B,aAAa,WAA6B;AAClE,UAAO,QAAQ,MAAA,yBAA+B,MAAM;IACpD;AACF,8BAA4B,aAAa,WAA6B;AACpE,UAAO,QAAQ,MAAA,2BAAiC,MAAM;IACtD;AACF,0BAAwB,aAAa,WAA6B;GAChE,MAAM,qBAAqB,MAAA,4BAAkC,IAAI;AACjE,UAAO,QAAQ,oBAAoB,MAAM;IACzC;;CAGJ,eAAe,MAAyB,QAAQ,GAAG;AACjD,MAAI,SAAS,OACX,OAAA,sBAA4B;MAE5B,OAAA,wBAA8B;;CAIlC,YAAY,MAAyB,QAAQ,GAAG;AAC9C,MAAI,SAAS,OACX,OAAA,oBAA0B;MAE1B,OAAA,sBAA4B;;CAIhC,iBAAiB,OAAe;AAC9B,QAAA,mBAAyB;;CAG3B,0BAA0B;AACxB,QAAA;;CAGF,4BAA4B;AAC1B,QAAA;;CAGF,4BAA4B,QAAsB;AAChD,QAAA,2BAAiC;;CAGnC,qBAAqB,QAA2B;AAC9C,QAAA,oBAA0B;;CAG5B,WAAW;AACT,QAAA,UAAgB;AAChB,MAAI,MAAA,eAAqB;AACvB,SAAA,IAAU,OAAO,2BAA2B;AACvC,SAAA,cAAoB,UAAU;;;CAIvC,iBAAiB;AACf,MAAI,CAAC,MAAA,kBAAwB;AAC3B,SAAA,mBAAyB;IACvB,eAAe,IAAI,MAAA,QAAc,SAAS,MAAM,UAAU,CAAC,UAAU;IACrE,mBAAmB,UAAU;IAC7B,uBAAuB;IACvB,uBAAuB,MAAA,aAAmB;IAC1C,gBAAgB,iBAAiB,MAAA,OAAa;IAC9C,gBAAgB,MAAA,QAAc,UAAU;IACxC,mBAAmB,MAAA,iBAAuB;IAC1C,mBAAmB,MAAA;IACnB,cAAc,MAAA,cAAoB;IACnC;AACD,SAAA,IAAU,QACR,gCAAgC,KAAK,UAAU,MAAA,iBAAuB,GACvE;;AAEH,SAAO,MAAA;;CAGT,eAAuB;AACrB,MAAI,QAAQ,IAAI,mBAAoB,QAAO;AAC3C,MAAI,QAAQ,IAAI,gBAAgB,QAAQ,IAAI,WAAY,QAAO;AAC/D,MACE,QAAQ,IAAI,iCACZ,QAAQ,IAAI,8BACZ,QAAQ,IAAI,kBAEZ,QAAO;AAET,MAAI,QAAQ,IAAI,eAAe,QAAQ,IAAI,mBACzC,QAAO;AAET,MAAI,QAAQ,IAAI,UAAU,QAAQ,IAAI,kBAAmB,QAAO;AAChE,MACE,QAAQ,IAAI,eACZ,QAAQ,IAAI,kBACZ,QAAQ,IAAI,qBAEZ,QAAO;AAET,MAAI,QAAQ,IAAI,eAAe,QAAQ,IAAI,uBACzC,QAAO;AAET,MAAI,QAAQ,IAAI,uBAAwB,QAAO;AAC/C,MAAI,QAAQ,IAAI,iBAAiB,QAAQ,IAAI,OAAQ,QAAO;AAC5D,SAAO;;CAGT,QAAQ,UAAkB,QAA+B;EACvD,IAAI,MAAM;AACV,SAAO,QAAQ,QAAQ,IAAI,EAAE;AAC3B,OAAI,WAAW,KAAK,KAAK,OAAO,CAAC,CAAE,QAAO;AAC1C,SAAM,QAAQ,IAAI;;AAEpB,SAAO;;CAGT,mBAA2B;AACzB,MAAI;GACF,MAAM,MAAM,QAAQ,KAAK;GACzB,MAAM,UAAU,MAAA,OAAa,KAAK,OAAO;AACzC,OAAI,CAAC,QACH,QAAO;GAGT,MAAM,iBAAiB,SAAS,wCAAwC;IACtE,KAAK;IACL,UAAU;IACV,SAAS;IACT,OAAO;KAAC;KAAU;KAAQ;KAAS;IACpC,CAAC,CAAC,MAAM;AAET,UAAO,eAAe,WAAW,KAAK,iBAAiB;WAChD,OAAO;GAKd,MAAM,UACJ,iBAAiB,QACb;IAAC,SAAS,MAAM;IAAS,MAAM,MAAM;IAAM,OAAO,MAAM;IAAM,GAC9D,OAAO,MAAM;AACnB,SAAA,IAAU,QAAQ,6CAA6C,QAAQ;AACvE,UAAO;;;CAIX,gBAAwB;AACtB,MAAI;AACF,OAAI,MAAA,eAAqB,CACvB,QAAO;GAET,MAAM,WAAW,KAAK,SAAS,EAAE,aAAa,OAAO;AAGrD,aAFgB,QAAQ,SAAS,EAEd,EAAC,WAAW,MAAK,CAAC;GAGrC,MAAM,QAAQ,YAAY;AAC1B,OAAI;AACF,kBAAc,UAAU,OAAO;KAAC,UAAU;KAAQ,MAAM;KAAK,CAAC;AAC9D,WAAO;YACA,YAAY;AACnB,QAAK,WAAqC,SAAS,SAEjD,QADmB,aAAa,UAAU,OAAO,CAAC,MAAM;AAG1D,UAAM;;WAED,OAAO;AACd,SAAA,IAAU,QACR,kDACA,MACD;AACD,UAAO;;;CAIX,iBAA0B;AACxB,MAAI;AACF,OAAI,QAAQ,IAAI,kBACd,QAAO;AAGT,OAAI,WAAW,cAAc,CAC3B,QAAO;AAGT,OAAI,WAAW,sCAAsC,CACnD,QAAO;AAGT,OAAI,QAAQ,IAAI,wBACd,QAAO;AAGT,OACE,QAAQ,IAAI,uBACZ,QAAQ,IAAI,UAAU,MAAM,WAAW,CAEvC,QAAO;AAGT,OAAI,WAAW,iBAAiB,EAAE;IAChC,MAAM,SAAS,aAAa,kBAAkB,OAAO;AACrD,QACE,OAAO,SAAS,SAAS,IACzB,OAAO,SAAS,WAAW,IAC3B,OAAO,SAAS,aAAa,CAE7B,QAAO;;AAIX,UAAO;WACA,OAAO;AACd,SAAA,IAAU,QACR,sDACA,MACD;AACD,UAAO;;;;AAKb,IAAM,gBAAgB,0BAA0B,aAAa;AAE7D,IAAa,2BAA2B,IAAiB,WACvD,SAAS,CAAC,MAAM,IAAI,OAAO;AAC7B,IAAa,kBAAkB,MAAyB,QAAQ,MAC9D,SAAS,CAAC,eAAe,MAAM,MAAM;AACvC,IAAa,eAAe,MAAyB,QAAQ,MAC3D,SAAS,CAAC,YAAY,MAAM,MAAM;AACpC,IAAa,oBAAoB,UAC/B,SAAS,CAAC,iBAAiB,MAAM;AACnC,IAAa,gCACX,SAAS,CAAC,yBAAyB;AACrC,IAAa,kCACX,SAAS,CAAC,2BAA2B;AACvC,IAAa,+BAA+B,WAC1C,SAAS,CAAC,4BAA4B,OAAO;AAC/C,IAAa,wBAAwB,WACnC,SAAS,CAAC,qBAAqB,OAAO"}
+{"version":3,"file":"anonymous-otel-start.js","names":["#instance","#processId","#lc","#starting","#config","#viewSyncerCount","#cachedAttributes","#run","#stopped","#getAttributes","#meterProvider","#meter","#setupMetrics","#activeUsersGetter","#totalCrudMutations","#totalCustomMutations","#totalCrudQueries","#totalCustomQueries","#totalRowsSynced","#totalConnectionsSuccess","#totalConnectionsAttempted","#activeClientGroupsGetter","#getPlatform","#getGitProjectId","#getOrSetFsID","#findUp","#isInContainer"],"sources":["../../../../../zero-cache/src/server/anonymous-otel-start.ts"],"sourcesContent":["import {execSync} from 'child_process';\nimport {randomUUID} from 'crypto';\nimport {existsSync, mkdirSync, readFileSync, writeFileSync} from 'fs';\nimport {homedir, platform} from 'os';\nimport {dirname, join} from 'path';\nimport type {ObservableResult} from '@opentelemetry/api';\nimport {type Meter} from '@opentelemetry/api';\nimport {OTLPMetricExporter} from '@opentelemetry/exporter-metrics-otlp-http';\nimport {resourceFromAttributes} from '@opentelemetry/resources';\nimport {\n  MeterProvider,\n  PeriodicExportingMetricReader,\n} from '@opentelemetry/sdk-metrics';\nimport type {LogContext} from '@rocicorp/logger';\nimport {h64} from '../../../shared/src/hash.js';\nimport {\n  getServerVersion,\n  getZeroConfig,\n  type ZeroConfig,\n} from '../config/zero-config.js';\nimport {setupOtelDiagnosticLogger} from './otel-diag-logger.ts';\n\nexport type ActiveUsers = {\n  active_users_last_day: number;\n  users_1da: number;\n  users_7da: number;\n  users_30da: number;\n  users_1da_legacy: number;\n  users_7da_legacy: number;\n  users_30da_legacy: number;\n};\n\nconst hostNameRe = /^[a-f0-9]{12}$/;\n\nclass AnonymousTelemetryManager {\n  static #instance: AnonymousTelemetryManager;\n  #starting = false;\n  #stopped = false;\n  #meter!: Meter;\n  #meterProvider!: MeterProvider;\n  #totalCrudMutations = 0;\n  #totalCustomMutations = 0;\n  #totalCrudQueries = 0;\n  #totalCustomQueries = 0;\n  #totalRowsSynced = 0;\n  #totalConnectionsSuccess = 0;\n  #totalConnectionsAttempted = 0;\n  #activeClientGroupsGetter: (() => number) | undefined;\n  #activeUsersGetter: (() => ActiveUsers) | undefined;\n  #lc: LogContext | undefined;\n  #config: ZeroConfig | undefined;\n  #processId: string;\n  #cachedAttributes: Record<string, string> | undefined;\n  #viewSyncerCount = 1;\n\n  private constructor() {\n    this.#processId = randomUUID();\n  }\n\n  static getInstance(): AnonymousTelemetryManager {\n    if (!AnonymousTelemetryManager.#instance) {\n      AnonymousTelemetryManager.#instance = new AnonymousTelemetryManager();\n    }\n    return AnonymousTelemetryManager.#instance;\n  }\n\n  start(lc?: LogContext, config?: ZeroConfig) {\n    this.#lc = lc;\n\n    // Set up OpenTelemetry diagnostic logger if not already configured\n    setupOtelDiagnosticLogger(lc);\n\n    if (!config) {\n      try {\n        config = getZeroConfig();\n      } catch (e) {\n        this.#lc?.info?.('telemetry: disabled - unable to parse config', e);\n        return;\n      }\n    }\n\n    if (process.env.DO_NOT_TRACK) {\n      this.#lc?.info?.(\n        'telemetry: disabled - DO_NOT_TRACK environment variable is set',\n      );\n      return;\n    }\n\n    if (!config.enableTelemetry) {\n      this.#lc?.info?.('telemetry: disabled - enableTelemetry is false');\n      return;\n    }\n\n    if (this.#starting) {\n      return;\n    }\n\n    this.#starting = true;\n    this.#config = config;\n    this.#viewSyncerCount = config.numSyncWorkers ?? 1;\n    this.#cachedAttributes = undefined;\n\n    this.#lc?.info?.(`telemetry: starting in 1 minute`);\n\n    // Delay telemetry startup by 1 minute to avoid potential boot loop issues\n    setTimeout(() => this.#run(), 60000);\n  }\n\n  #run() {\n    if (this.#stopped) {\n      return;\n    }\n\n    const resource = resourceFromAttributes(this.#getAttributes());\n\n    // Add a random jitter to the export interval to avoid all view-syncers exporting at the same time\n    const exportIntervalMillis =\n      60000 * this.#viewSyncerCount + Math.floor(Math.random() * 10000);\n    const readers = [\n      new PeriodicExportingMetricReader({\n        exportIntervalMillis,\n        exporter: new OTLPMetricExporter({\n          url: 'https://metrics.rocicorp.dev',\n          timeoutMillis: 30000,\n        }),\n      }),\n    ];\n\n    // Uncomment this to debug metrics exports.\n\n    // readers.push(\n    //   new PeriodicExportingMetricReader({\n    //     exportIntervalMillis,\n    //     exporter: new ConsoleMetricExporter(),\n    //   }),\n    // );\n\n    this.#meterProvider = new MeterProvider({resource, readers});\n    this.#meter = this.#meterProvider.getMeter('zero-anonymous-telemetry');\n\n    this.#setupMetrics();\n    this.#lc?.info?.(\n      `telemetry: started (exports every ${exportIntervalMillis / 1000} seconds for ${this.#viewSyncerCount} view-syncers)`,\n    );\n  }\n\n  #setupMetrics() {\n    // Observable gauges\n    const uptimeGauge = this.#meter.createObservableGauge('zero.uptime', {\n      description: 'System uptime in seconds',\n      unit: 'seconds',\n    });\n\n    // Observable counters\n    const uptimeCounter = this.#meter.createObservableCounter(\n      'zero.uptime_counter',\n      {\n        description: 'System uptime in seconds',\n        unit: 'seconds',\n      },\n    );\n    const crudMutationsCounter = this.#meter.createObservableCounter(\n      'zero.crud_mutations_processed',\n      {\n        description: 'Total number of CRUD mutations processed',\n      },\n    );\n    const customMutationsCounter = this.#meter.createObservableCounter(\n      'zero.custom_mutations_processed',\n      {\n        description: 'Total number of custom mutations processed',\n      },\n    );\n    const totalMutationsCounter = this.#meter.createObservableCounter(\n      'zero.mutations_processed',\n      {\n        description: 'Total number of mutations processed',\n      },\n    );\n    const crudQueriesCounter = this.#meter.createObservableCounter(\n      'zero.crud_queries_processed',\n      {\n        description: 'Total number of CRUD queries processed',\n      },\n    );\n    const customQueriesCounter = this.#meter.createObservableCounter(\n      'zero.custom_queries_processed',\n      {\n        description: 'Total number of custom queries processed',\n      },\n    );\n    const totalQueriesCounter = this.#meter.createObservableCounter(\n      'zero.queries_processed',\n      {\n        description: 'Total number of queries processed',\n      },\n    );\n    const rowsSyncedCounter = this.#meter.createObservableCounter(\n      'zero.rows_synced',\n      {\n        description: 'Total number of rows synced',\n      },\n    );\n\n    // Observable counters for connections\n    const connectionsSuccessCounter = this.#meter.createObservableCounter(\n      'zero.connections_success',\n      {\n        description: 'Total number of successful connections',\n      },\n    );\n\n    const connectionsAttemptedCounter = this.#meter.createObservableCounter(\n      'zero.connections_attempted',\n      {\n        description: 'Total number of attempted connections',\n      },\n    );\n\n    const activeClientGroupsGauge = this.#meter.createObservableGauge(\n      'zero.gauge_active_client_groups',\n      {\n        description: 'Number of currently active client groups',\n      },\n    );\n\n    const attrs = this.#getAttributes();\n    const active =\n      (metric: keyof ActiveUsers) => (result: ObservableResult) => {\n        const actives = this.#activeUsersGetter?.();\n        if (actives) {\n          const value = actives[metric];\n          result.observe(value, attrs);\n        }\n      };\n    this.#meter\n      .createObservableGauge('zero.active_users_last_day', {\n        description: 'Count of CVR instances active in the last 24h',\n      })\n      .addCallback(active('active_users_last_day'));\n    this.#meter\n      .createObservableGauge('zero.users_1da', {\n        description: 'Count of 1-day active profiles',\n      })\n      .addCallback(active('users_1da'));\n    this.#meter\n      .createObservableGauge('zero.users_7da', {\n        description: 'Count of 7-day active profiles',\n      })\n      .addCallback(active('users_7da'));\n    this.#meter\n      .createObservableGauge('zero.users_30da', {\n        description: 'Count of 30-day active profiles',\n      })\n      .addCallback(active('users_30da'));\n    this.#meter\n      .createObservableGauge('zero.users_1da_legacy', {\n        description: 'Count of 1-day active profiles with CVR fallback',\n      })\n      .addCallback(active('users_1da_legacy'));\n    this.#meter\n      .createObservableGauge('zero.users_7da_legacy', {\n        description: 'Count of 7-day active profiles with CVR fallback',\n      })\n      .addCallback(active('users_7da_legacy'));\n    this.#meter\n      .createObservableGauge('zero.users_30da_legacy', {\n        description: 'Count of 30-day active profiles with CVR fallback',\n      })\n      .addCallback(active('users_30da_legacy'));\n\n    // Callbacks\n    uptimeGauge.addCallback((result: ObservableResult) => {\n      const uptimeSeconds = Math.floor(process.uptime());\n      result.observe(uptimeSeconds, attrs);\n    });\n    uptimeCounter.addCallback((result: ObservableResult) => {\n      const uptimeSeconds = Math.floor(process.uptime());\n      result.observe(uptimeSeconds, attrs);\n    });\n    crudMutationsCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCrudMutations, attrs);\n    });\n    customMutationsCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCustomMutations, attrs);\n    });\n    totalMutationsCounter.addCallback((result: ObservableResult) => {\n      const totalMutations =\n        this.#totalCrudMutations + this.#totalCustomMutations;\n      result.observe(totalMutations, attrs);\n    });\n    crudQueriesCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCrudQueries, attrs);\n    });\n    customQueriesCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalCustomQueries, attrs);\n    });\n    totalQueriesCounter.addCallback((result: ObservableResult) => {\n      const totalQueries = this.#totalCrudQueries + this.#totalCustomQueries;\n      result.observe(totalQueries, attrs);\n    });\n    rowsSyncedCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalRowsSynced, attrs);\n    });\n    connectionsSuccessCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalConnectionsSuccess, attrs);\n    });\n    connectionsAttemptedCounter.addCallback((result: ObservableResult) => {\n      result.observe(this.#totalConnectionsAttempted, attrs);\n    });\n    activeClientGroupsGauge.addCallback((result: ObservableResult) => {\n      const activeClientGroups = this.#activeClientGroupsGetter?.() ?? 0;\n      result.observe(activeClientGroups, attrs);\n    });\n  }\n\n  recordMutation(type: 'crud' | 'custom', count = 1) {\n    if (type === 'crud') {\n      this.#totalCrudMutations += count;\n    } else {\n      this.#totalCustomMutations += count;\n    }\n  }\n\n  recordQuery(type: 'crud' | 'custom', count = 1) {\n    if (type === 'crud') {\n      this.#totalCrudQueries += count;\n    } else {\n      this.#totalCustomQueries += count;\n    }\n  }\n\n  recordRowsSynced(count: number) {\n    this.#totalRowsSynced += count;\n  }\n\n  recordConnectionSuccess() {\n    this.#totalConnectionsSuccess++;\n  }\n\n  recordConnectionAttempted() {\n    this.#totalConnectionsAttempted++;\n  }\n\n  setActiveClientGroupsGetter(getter: () => number) {\n    this.#activeClientGroupsGetter = getter;\n  }\n\n  setActiveUsersGetter(getter: () => ActiveUsers) {\n    this.#activeUsersGetter = getter;\n  }\n\n  shutdown() {\n    this.#stopped = true;\n    if (this.#meterProvider) {\n      this.#lc?.info?.('telemetry: shutting down');\n      void this.#meterProvider.shutdown();\n    }\n  }\n\n  #getAttributes() {\n    if (!this.#cachedAttributes) {\n      this.#cachedAttributes = {\n        'zero.app.id': h64(this.#config?.upstream.db || 'unknown').toString(),\n        'zero.machine.os': platform(),\n        'zero.telemetry.type': 'anonymous',\n        'zero.infra.platform': this.#getPlatform(),\n        'zero.version': getServerVersion(this.#config),\n        'zero.task.id': this.#config?.taskID || 'unknown',\n        'zero.project.id': this.#getGitProjectId(),\n        'zero.process.id': this.#processId,\n        'zero.fs.id': this.#getOrSetFsID(),\n      };\n      this.#lc?.debug?.(\n        `telemetry: cached attributes=${JSON.stringify(this.#cachedAttributes)}`,\n      );\n    }\n    return this.#cachedAttributes;\n  }\n\n  #getPlatform(): string {\n    if (process.env.ZERO_ON_CLOUD_ZERO) return 'cloudzero';\n    if (process.env.FLY_APP_NAME || process.env.FLY_REGION) return 'fly.io';\n    if (\n      process.env.ECS_CONTAINER_METADATA_URI_V4 ||\n      process.env.ECS_CONTAINER_METADATA_URI ||\n      process.env.AWS_EXECUTION_ENV\n    ) {\n      return 'aws';\n    }\n    if (process.env.RAILWAY_ENV || process.env.RAILWAY_STATIC_URL) {\n      return 'railway';\n    }\n    if (process.env.RENDER || process.env.RENDER_SERVICE_ID) return 'render';\n    if (\n      process.env.GCP_PROJECT ||\n      process.env.GCLOUD_PROJECT ||\n      process.env.GOOGLE_CLOUD_PROJECT\n    ) {\n      return 'gcp';\n    }\n    if (process.env.COOLIFY_URL || process.env.COOLIFY_CONTAINER_NAME) {\n      return 'coolify';\n    }\n    if (process.env.CONTAINER_APP_REVISION) return 'azure';\n    if (process.env.FLIGHTCONTROL || process.env.FC_URL) return 'flightcontrol';\n    return 'unknown';\n  }\n\n  #findUp(startDir: string, target: string): string | null {\n    let dir = startDir;\n    while (dir !== dirname(dir)) {\n      if (existsSync(join(dir, target))) return dir;\n      dir = dirname(dir);\n    }\n    return null;\n  }\n\n  #getGitProjectId(): string {\n    try {\n      const cwd = process.cwd();\n      const gitRoot = this.#findUp(cwd, '.git');\n      if (!gitRoot) {\n        return 'unknown';\n      }\n\n      const rootCommitHash = execSync('git rev-list --max-parents=0 HEAD -1', {\n        cwd: gitRoot,\n        encoding: 'utf8',\n        timeout: 1000,\n        stdio: ['ignore', 'pipe', 'ignore'], // Suppress stderr\n      }).trim();\n\n      return rootCommitHash.length === 40 ? rootCommitHash : 'unknown';\n    } catch (error) {\n      // execSync throws a child_process.SpawnSyncReturns-shaped error with an\n      // output property (an array with stdio buffers) and sometimes a killed\n      // property that references back to the error itself — making it\n      // circular and causing stringify() in the Logger logic to throw.\n      const details =\n        error instanceof Error\n          ? {message: error.message, name: error.name, stack: error.stack}\n          : String(error);\n      this.#lc?.debug?.('telemetry: unable to get Git root commit:', details);\n      return 'unknown';\n    }\n  }\n\n  #getOrSetFsID(): string {\n    try {\n      if (this.#isInContainer()) {\n        return 'container';\n      }\n      const fsidPath = join(homedir(), '.rocicorp', 'fsid');\n      const fsidDir = dirname(fsidPath);\n\n      mkdirSync(fsidDir, {recursive: true});\n\n      // Always try atomic file creation first - this eliminates any race conditions\n      const newId = randomUUID();\n      try {\n        writeFileSync(fsidPath, newId, {encoding: 'utf8', flag: 'wx'});\n        return newId;\n      } catch (writeError) {\n        if ((writeError as NodeJS.ErrnoException).code === 'EEXIST') {\n          const existingId = readFileSync(fsidPath, 'utf8').trim();\n          return existingId;\n        }\n        throw writeError;\n      }\n    } catch (error) {\n      this.#lc?.debug?.(\n        'telemetry: unable to get or set filesystem ID:',\n        error,\n      );\n      return 'unknown';\n    }\n  }\n\n  #isInContainer(): boolean {\n    try {\n      if (process.env.ZERO_IN_CONTAINER) {\n        return true;\n      }\n\n      if (existsSync('/.dockerenv')) {\n        return true;\n      }\n\n      if (existsSync('/usr/local/bin/docker-entrypoint.sh')) {\n        return true;\n      }\n\n      if (process.env.KUBERNETES_SERVICE_HOST) {\n        return true;\n      }\n\n      if (\n        process.env.DOCKER_CONTAINER_ID ||\n        process.env.HOSTNAME?.match(hostNameRe)\n      ) {\n        return true;\n      }\n\n      if (existsSync('/proc/1/cgroup')) {\n        const cgroup = readFileSync('/proc/1/cgroup', 'utf8');\n        if (\n          cgroup.includes('docker') ||\n          cgroup.includes('kubepods') ||\n          cgroup.includes('containerd')\n        ) {\n          return true;\n        }\n      }\n\n      return false;\n    } catch (error) {\n      this.#lc?.debug?.(\n        'telemetry: unable to detect container environment:',\n        error,\n      );\n      return false;\n    }\n  }\n}\n\nconst manager = () => AnonymousTelemetryManager.getInstance();\n\nexport const startAnonymousTelemetry = (lc?: LogContext, config?: ZeroConfig) =>\n  manager().start(lc, config);\nexport const recordMutation = (type: 'crud' | 'custom', count = 1) =>\n  manager().recordMutation(type, count);\nexport const recordQuery = (type: 'crud' | 'custom', count = 1) =>\n  manager().recordQuery(type, count);\nexport const recordRowsSynced = (count: number) =>\n  manager().recordRowsSynced(count);\nexport const recordConnectionSuccess = () =>\n  manager().recordConnectionSuccess();\nexport const recordConnectionAttempted = () =>\n  manager().recordConnectionAttempted();\nexport const setActiveClientGroupsGetter = (getter: () => number) =>\n  manager().setActiveClientGroupsGetter(getter);\nexport const setActiveUsersGetter = (getter: () => ActiveUsers) =>\n  manager().setActiveUsersGetter(getter);\nexport const shutdownAnonymousTelemetry = () => manager().shutdown();\n"],"mappings":";;;;;;;;;;;;;AAgCA,IAAM,aAAa;AAEnB,IAAM,4BAAN,MAAM,0BAA0B;CAC9B,QAAA;CACA,YAAY;CACZ,WAAW;CACX;CACA;CACA,sBAAsB;CACtB,wBAAwB;CACxB,oBAAoB;CACpB,sBAAsB;CACtB,mBAAmB;CACnB,2BAA2B;CAC3B,6BAA6B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CAEnB,cAAsB;AACpB,QAAA,YAAkB,YAAY;;CAGhC,OAAO,cAAyC;AAC9C,MAAI,CAAC,2BAAA,SACH,4BAAA,WAAsC,IAAI,2BAA2B;AAEvE,SAAO,2BAAA;;CAGT,MAAM,IAAiB,QAAqB;AAC1C,QAAA,KAAW;AAGX,4BAA0B,GAAG;AAE7B,MAAI,CAAC,OACH,KAAI;AACF,YAAS,eAAe;WACjB,GAAG;AACV,SAAA,IAAU,OAAO,gDAAgD,EAAE;AACnE;;AAIJ,MAAI,QAAQ,IAAI,cAAc;AAC5B,SAAA,IAAU,OACR,iEACD;AACD;;AAGF,MAAI,CAAC,OAAO,iBAAiB;AAC3B,SAAA,IAAU,OAAO,iDAAiD;AAClE;;AAGF,MAAI,MAAA,SACF;AAGF,QAAA,WAAiB;AACjB,QAAA,SAAe;AACf,QAAA,kBAAwB,OAAO,kBAAkB;AACjD,QAAA,mBAAyB,KAAA;AAEzB,QAAA,IAAU,OAAO,kCAAkC;AAGnD,mBAAiB,MAAA,KAAW,EAAE,IAAM;;CAGtC,OAAO;AACL,MAAI,MAAA,QACF;EAGF,MAAM,WAAW,uBAAuB,MAAA,eAAqB,CAAC;EAG9D,MAAM,uBACJ,MAAQ,MAAA,kBAAwB,KAAK,MAAM,KAAK,QAAQ,GAAG,IAAM;AAoBnE,QAAA,gBAAsB,IAAI,cAAc;GAAC;GAAU,SAnBnC,CACd,IAAI,8BAA8B;IAChC;IACA,UAAU,IAAI,mBAAmB;KAC/B,KAAK;KACL,eAAe;KAChB,CAAC;IACH,CAAC,CACH;GAW0D,CAAC;AAC5D,QAAA,QAAc,MAAA,cAAoB,SAAS,2BAA2B;AAEtE,QAAA,cAAoB;AACpB,QAAA,IAAU,OACR,qCAAqC,uBAAuB,IAAK,eAAe,MAAA,gBAAsB,gBACvG;;CAGH,gBAAgB;EAEd,MAAM,cAAc,MAAA,MAAY,sBAAsB,eAAe;GACnE,aAAa;GACb,MAAM;GACP,CAAC;EAGF,MAAM,gBAAgB,MAAA,MAAY,wBAChC,uBACA;GACE,aAAa;GACb,MAAM;GACP,CACF;EACD,MAAM,uBAAuB,MAAA,MAAY,wBACvC,iCACA,EACE,aAAa,4CACd,CACF;EACD,MAAM,yBAAyB,MAAA,MAAY,wBACzC,mCACA,EACE,aAAa,8CACd,CACF;EACD,MAAM,wBAAwB,MAAA,MAAY,wBACxC,4BACA,EACE,aAAa,uCACd,CACF;EACD,MAAM,qBAAqB,MAAA,MAAY,wBACrC,+BACA,EACE,aAAa,0CACd,CACF;EACD,MAAM,uBAAuB,MAAA,MAAY,wBACvC,iCACA,EACE,aAAa,4CACd,CACF;EACD,MAAM,sBAAsB,MAAA,MAAY,wBACtC,0BACA,EACE,aAAa,qCACd,CACF;EACD,MAAM,oBAAoB,MAAA,MAAY,wBACpC,oBACA,EACE,aAAa,+BACd,CACF;EAGD,MAAM,4BAA4B,MAAA,MAAY,wBAC5C,4BACA,EACE,aAAa,0CACd,CACF;EAED,MAAM,8BAA8B,MAAA,MAAY,wBAC9C,8BACA,EACE,aAAa,yCACd,CACF;EAED,MAAM,0BAA0B,MAAA,MAAY,sBAC1C,mCACA,EACE,aAAa,4CACd,CACF;EAED,MAAM,QAAQ,MAAA,eAAqB;EACnC,MAAM,UACH,YAA+B,WAA6B;GAC3D,MAAM,UAAU,MAAA,qBAA2B;AAC3C,OAAI,SAAS;IACX,MAAM,QAAQ,QAAQ;AACtB,WAAO,QAAQ,OAAO,MAAM;;;AAGlC,QAAA,MACG,sBAAsB,8BAA8B,EACnD,aAAa,iDACd,CAAC,CACD,YAAY,OAAO,wBAAwB,CAAC;AAC/C,QAAA,MACG,sBAAsB,kBAAkB,EACvC,aAAa,kCACd,CAAC,CACD,YAAY,OAAO,YAAY,CAAC;AACnC,QAAA,MACG,sBAAsB,kBAAkB,EACvC,aAAa,kCACd,CAAC,CACD,YAAY,OAAO,YAAY,CAAC;AACnC,QAAA,MACG,sBAAsB,mBAAmB,EACxC,aAAa,mCACd,CAAC,CACD,YAAY,OAAO,aAAa,CAAC;AACpC,QAAA,MACG,sBAAsB,yBAAyB,EAC9C,aAAa,oDACd,CAAC,CACD,YAAY,OAAO,mBAAmB,CAAC;AAC1C,QAAA,MACG,sBAAsB,yBAAyB,EAC9C,aAAa,oDACd,CAAC,CACD,YAAY,OAAO,mBAAmB,CAAC;AAC1C,QAAA,MACG,sBAAsB,0BAA0B,EAC/C,aAAa,qDACd,CAAC,CACD,YAAY,OAAO,oBAAoB,CAAC;AAG3C,cAAY,aAAa,WAA6B;GACpD,MAAM,gBAAgB,KAAK,MAAM,QAAQ,QAAQ,CAAC;AAClD,UAAO,QAAQ,eAAe,MAAM;IACpC;AACF,gBAAc,aAAa,WAA6B;GACtD,MAAM,gBAAgB,KAAK,MAAM,QAAQ,QAAQ,CAAC;AAClD,UAAO,QAAQ,eAAe,MAAM;IACpC;AACF,uBAAqB,aAAa,WAA6B;AAC7D,UAAO,QAAQ,MAAA,oBAA0B,MAAM;IAC/C;AACF,yBAAuB,aAAa,WAA6B;AAC/D,UAAO,QAAQ,MAAA,sBAA4B,MAAM;IACjD;AACF,wBAAsB,aAAa,WAA6B;GAC9D,MAAM,iBACJ,MAAA,qBAA2B,MAAA;AAC7B,UAAO,QAAQ,gBAAgB,MAAM;IACrC;AACF,qBAAmB,aAAa,WAA6B;AAC3D,UAAO,QAAQ,MAAA,kBAAwB,MAAM;IAC7C;AACF,uBAAqB,aAAa,WAA6B;AAC7D,UAAO,QAAQ,MAAA,oBAA0B,MAAM;IAC/C;AACF,sBAAoB,aAAa,WAA6B;GAC5D,MAAM,eAAe,MAAA,mBAAyB,MAAA;AAC9C,UAAO,QAAQ,cAAc,MAAM;IACnC;AACF,oBAAkB,aAAa,WAA6B;AAC1D,UAAO,QAAQ,MAAA,iBAAuB,MAAM;IAC5C;AACF,4BAA0B,aAAa,WAA6B;AAClE,UAAO,QAAQ,MAAA,yBAA+B,MAAM;IACpD;AACF,8BAA4B,aAAa,WAA6B;AACpE,UAAO,QAAQ,MAAA,2BAAiC,MAAM;IACtD;AACF,0BAAwB,aAAa,WAA6B;GAChE,MAAM,qBAAqB,MAAA,4BAAkC,IAAI;AACjE,UAAO,QAAQ,oBAAoB,MAAM;IACzC;;CAGJ,eAAe,MAAyB,QAAQ,GAAG;AACjD,MAAI,SAAS,OACX,OAAA,sBAA4B;MAE5B,OAAA,wBAA8B;;CAIlC,YAAY,MAAyB,QAAQ,GAAG;AAC9C,MAAI,SAAS,OACX,OAAA,oBAA0B;MAE1B,OAAA,sBAA4B;;CAIhC,iBAAiB,OAAe;AAC9B,QAAA,mBAAyB;;CAG3B,0BAA0B;AACxB,QAAA;;CAGF,4BAA4B;AAC1B,QAAA;;CAGF,4BAA4B,QAAsB;AAChD,QAAA,2BAAiC;;CAGnC,qBAAqB,QAA2B;AAC9C,QAAA,oBAA0B;;CAG5B,WAAW;AACT,QAAA,UAAgB;AAChB,MAAI,MAAA,eAAqB;AACvB,SAAA,IAAU,OAAO,2BAA2B;AACvC,SAAA,cAAoB,UAAU;;;CAIvC,iBAAiB;AACf,MAAI,CAAC,MAAA,kBAAwB;AAC3B,SAAA,mBAAyB;IACvB,eAAe,IAAI,MAAA,QAAc,SAAS,MAAM,UAAU,CAAC,UAAU;IACrE,mBAAmB,UAAU;IAC7B,uBAAuB;IACvB,uBAAuB,MAAA,aAAmB;IAC1C,gBAAgB,iBAAiB,MAAA,OAAa;IAC9C,gBAAgB,MAAA,QAAc,UAAU;IACxC,mBAAmB,MAAA,iBAAuB;IAC1C,mBAAmB,MAAA;IACnB,cAAc,MAAA,cAAoB;IACnC;AACD,SAAA,IAAU,QACR,gCAAgC,KAAK,UAAU,MAAA,iBAAuB,GACvE;;AAEH,SAAO,MAAA;;CAGT,eAAuB;AACrB,MAAI,QAAQ,IAAI,mBAAoB,QAAO;AAC3C,MAAI,QAAQ,IAAI,gBAAgB,QAAQ,IAAI,WAAY,QAAO;AAC/D,MACE,QAAQ,IAAI,iCACZ,QAAQ,IAAI,8BACZ,QAAQ,IAAI,kBAEZ,QAAO;AAET,MAAI,QAAQ,IAAI,eAAe,QAAQ,IAAI,mBACzC,QAAO;AAET,MAAI,QAAQ,IAAI,UAAU,QAAQ,IAAI,kBAAmB,QAAO;AAChE,MACE,QAAQ,IAAI,eACZ,QAAQ,IAAI,kBACZ,QAAQ,IAAI,qBAEZ,QAAO;AAET,MAAI,QAAQ,IAAI,eAAe,QAAQ,IAAI,uBACzC,QAAO;AAET,MAAI,QAAQ,IAAI,uBAAwB,QAAO;AAC/C,MAAI,QAAQ,IAAI,iBAAiB,QAAQ,IAAI,OAAQ,QAAO;AAC5D,SAAO;;CAGT,QAAQ,UAAkB,QAA+B;EACvD,IAAI,MAAM;AACV,SAAO,QAAQ,QAAQ,IAAI,EAAE;AAC3B,OAAI,WAAW,KAAK,KAAK,OAAO,CAAC,CAAE,QAAO;AAC1C,SAAM,QAAQ,IAAI;;AAEpB,SAAO;;CAGT,mBAA2B;AACzB,MAAI;GACF,MAAM,MAAM,QAAQ,KAAK;GACzB,MAAM,UAAU,MAAA,OAAa,KAAK,OAAO;AACzC,OAAI,CAAC,QACH,QAAO;GAGT,MAAM,iBAAiB,SAAS,wCAAwC;IACtE,KAAK;IACL,UAAU;IACV,SAAS;IACT,OAAO;KAAC;KAAU;KAAQ;KAAS;IACpC,CAAC,CAAC,MAAM;AAET,UAAO,eAAe,WAAW,KAAK,iBAAiB;WAChD,OAAO;GAKd,MAAM,UACJ,iBAAiB,QACb;IAAC,SAAS,MAAM;IAAS,MAAM,MAAM;IAAM,OAAO,MAAM;IAAM,GAC9D,OAAO,MAAM;AACnB,SAAA,IAAU,QAAQ,6CAA6C,QAAQ;AACvE,UAAO;;;CAIX,gBAAwB;AACtB,MAAI;AACF,OAAI,MAAA,eAAqB,CACvB,QAAO;GAET,MAAM,WAAW,KAAK,SAAS,EAAE,aAAa,OAAO;AAGrD,aAFgB,QAAQ,SAAS,EAEd,EAAC,WAAW,MAAK,CAAC;GAGrC,MAAM,QAAQ,YAAY;AAC1B,OAAI;AACF,kBAAc,UAAU,OAAO;KAAC,UAAU;KAAQ,MAAM;KAAK,CAAC;AAC9D,WAAO;YACA,YAAY;AACnB,QAAK,WAAqC,SAAS,SAEjD,QADmB,aAAa,UAAU,OAAO,CAAC,MAAM;AAG1D,UAAM;;WAED,OAAO;AACd,SAAA,IAAU,QACR,kDACA,MACD;AACD,UAAO;;;CAIX,iBAA0B;AACxB,MAAI;AACF,OAAI,QAAQ,IAAI,kBACd,QAAO;AAGT,OAAI,WAAW,cAAc,CAC3B,QAAO;AAGT,OAAI,WAAW,sCAAsC,CACnD,QAAO;AAGT,OAAI,QAAQ,IAAI,wBACd,QAAO;AAGT,OACE,QAAQ,IAAI,uBACZ,QAAQ,IAAI,UAAU,MAAM,WAAW,CAEvC,QAAO;AAGT,OAAI,WAAW,iBAAiB,EAAE;IAChC,MAAM,SAAS,aAAa,kBAAkB,OAAO;AACrD,QACE,OAAO,SAAS,SAAS,IACzB,OAAO,SAAS,WAAW,IAC3B,OAAO,SAAS,aAAa,CAE7B,QAAO;;AAIX,UAAO;WACA,OAAO;AACd,SAAA,IAAU,QACR,sDACA,MACD;AACD,UAAO;;;;AAKb,IAAM,gBAAgB,0BAA0B,aAAa;AAE7D,IAAa,2BAA2B,IAAiB,WACvD,SAAS,CAAC,MAAM,IAAI,OAAO;AAC7B,IAAa,kBAAkB,MAAyB,QAAQ,MAC9D,SAAS,CAAC,eAAe,MAAM,MAAM;AACvC,IAAa,eAAe,MAAyB,QAAQ,MAC3D,SAAS,CAAC,YAAY,MAAM,MAAM;AACpC,IAAa,oBAAoB,UAC/B,SAAS,CAAC,iBAAiB,MAAM;AACnC,IAAa,gCACX,SAAS,CAAC,yBAAyB;AACrC,IAAa,kCACX,SAAS,CAAC,2BAA2B;AACvC,IAAa,+BAA+B,WAC1C,SAAS,CAAC,4BAA4B,OAAO;AAC/C,IAAa,wBAAwB,WACnC,SAAS,CAAC,qBAAqB,OAAO"}

package/out/zero-cache/src/server/change-streamer.d.ts

@@ -1,3 +1,3 @@
 import { type Worker } from '../types/processes.ts';
-export default function runWorker(parent: Worker, env: NodeJS.ProcessEnv, ...args: string[]): Promise<void>;
+export default function runWorker(parent: Worker, env: NodeJS.ProcessEnv, ...argv: string[]): Promise<void>;
 //# sourceMappingURL=change-streamer.d.ts.map

package/out/zero-cache/src/server/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA0BA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAyJf"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAgCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA0Lf"}

package/out/zero-cache/src/server/change-streamer.js

@@ -1,13 +1,12 @@
 import { assert } from "../../../shared/src/asserts.js";
 import { must } from "../../../shared/src/must.js";
-import { DatabaseInitError } from "../../../zqlite/src/db.js";
 import { parentWorker, singleProcessMode } from "../types/processes.js";
-import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
-import { startOtelAuto } from "./otel-start.js";
-import { createLogContext } from "./logging.js";
+import { DatabaseInitError } from "../../../zqlite/src/db.js";
 import { pgClient } from "../types/pg.js";
+import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
+import { createLogContext } from "./logging.js";
 import { getServerContext } from "../config/server-context.js";
 import { deleteLiteDB } from "../db/delete-lite-db.js";
 import { warmupConnections } from "../db/warmup.js";
@@ -19,19 +18,22 @@ import { initializeCustomChangeSource } from "../services/change-source/custom/c
 import { initializePostgresChangeSource } from "../services/change-source/pg/change-source.js";
 import { BackupMonitor } from "../services/change-streamer/backup-monitor.js";
 import { ChangeStreamerHttpServer } from "../services/change-streamer/change-streamer-http.js";
+import { initChangeStreamerSchema } from "../services/change-streamer/schema/init.js";
+import { PurgeLocker } from "../services/change-streamer/storer.js";
 import { initializeStreamer } from "../services/change-streamer/change-streamer-service.js";
 import { ReplicaMonitor } from "../services/change-streamer/replica-monitor.js";
+import { BackupNotFoundException, restoreReplica } from "../services/litestream/commands.js";
+import { startOtelAuto } from "./otel-start.js";
 //#region ../zero-cache/src/server/change-streamer.ts
-async function runWorker(parent, env, ...args) {
-	assert(args.length > 0, `parent startMs not specified`);
-	const parentStartMs = parseInt(args[0]);
+async function runWorker(parent, env, ...argv) {
+	const workerStartTime = Date.now();
 	const config = getNormalizedZeroConfig({
 		env,
-		argv: args.slice(1)
+		argv
 	});
 	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds }, upstream, change, replica, initialSync, litestream } = config;
-	startOtelAuto(createLogContext(config, { worker: "change-streamer" }, false));
-	const lc = createLogContext(config, { worker: "change-streamer" }, true);
+	startOtelAuto(createLogContext(config, "change-streamer", 0, false), "change-streamer", 0);
+	const lc = createLogContext(config, "change-streamer");
 	initEventSink(lc, config);
 	const changeDB = pgClient(lc, change.db, {
 		max: change.maxConns,
@@ -40,6 +42,15 @@ async function runWorker(parent, env, ...args) {
 	warmupConnections(lc, changeDB, "change").catch(() => {});
 	const { autoReset, replicationLag } = config;
 	const shard = getShardConfig(config);
+	await initChangeStreamerSchema(lc, changeDB, shard);
+	let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();
+	if (purgeLock) try {
+		await restoreReplica(lc, config, purgeLock);
+	} catch (e) {
+		lc[e instanceof BackupNotFoundException ? "warn" : "error"]?.(`error restoring backup. resyncing the replica: ${String(e)}`, e);
+		await purgeLock.release();
+		purgeLock = null;
+	}
 	let changeStreamer;
 	const context = getServerContext(config);
 	for (const first of [true, false]) try {
@@ -47,7 +58,11 @@ async function runWorker(parent, env, ...args) {
 			...initialSync,
 			replicationSlotFailover: upstream.pgReplicationSlotFailover
 		}, context, replicationLag.reportIntervalMs) : await initializeCustomChangeSource(lc, upstream.db, shard, replica.file, context);
-		changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, ReplicationStatusPublisher.forReplicaFile(replica.file), subscriptionState, autoReset ?? false, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeout);
+		changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, ReplicationStatusPublisher.forReplicaFile(replica.file), subscriptionState, purgeLock, autoReset ?? false, {
+			backPressureLimitHeapProportion,
+			flowControlConsensusPaddingSeconds,
+			statementTimeoutMs: change.statementTimeoutMs
+		}, setTimeout);
 		break;
 	} catch (e) {
 		if (first && e instanceof AutoResetSignal) {
@@ -62,7 +77,7 @@ async function runWorker(parent, env, ...args) {
 	assert(changeStreamer, `resetting replica did not advance replicaVersion`);
 	await upgradeReplica(lc, "change-streamer-init", replica.file);
 	const { backupURL, port: metricsPort } = litestream;
-	const monitor = backupURL ? new BackupMonitor(lc, replica.file, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, Date.now() - parentStartMs) : new ReplicaMonitor(lc, replica.file, changeStreamer);
+	const monitor = backupURL ? new BackupMonitor(lc, replica.file, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, Date.now() - workerStartTime) : new ReplicaMonitor(lc, replica.file, changeStreamer);
 	const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, config, {
 		port,
 		startupDelayMs

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {\n  AutoResetSignal,\n  CHANGE_STREAMER_APP_NAME,\n} from '../services/change-streamer/schema/tables.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `parent startMs not specified`);\n  const parentStartMs = parseInt(args[0]);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusPaddingSeconds,\n    },\n    upstream,\n    change,\n    replica,\n    initialSync,\n    litestream,\n  } = config;\n\n  startOtelAuto(createLogContext(config, {worker: 'change-streamer'}, false));\n  const lc = createLogContext(config, {worker: 'change-streamer'}, true);\n  initEventSink(lc, config);\n\n  // Kick off DB connection warmup in the background.\n  const changeDB = pgClient(\n    lc,\n    change.db,\n    {\n      max: change.maxConns,\n      connection: {['application_name']: CHANGE_STREAMER_APP_NAME},\n    },\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const {autoReset, replicationLag} = config;\n  const shard = getShardConfig(config);\n\n  let changeStreamer: ChangeStreamerService | undefined;\n\n  const context = getServerContext(config);\n\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {changeSource, subscriptionState} =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        autoReset ?? false,\n        backPressureLimitHeapProportion,\n        flowControlConsensusPaddingSeconds,\n        setTimeout,\n      );\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      await publishCriticalEvent(\n        lc,\n        replicationStatusError(lc, 'Initializing', e),\n      );\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  // Perform any upgrades to the replica in case it was restored from an\n  // earlier version. Note that this upgrade is done by the replicator worker\n  // as well (in both the replication-manager and the view-syncer), but the\n  // change-streamer independently reads the replica, and it is fine run the\n  // upgrade logic redundantly since it is idempotent.\n  await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n  const {backupURL, port: metricsPort} = litestream;\n  const monitor = backupURL\n    ? new BackupMonitor(\n        lc,\n        replica.file,\n        backupURL,\n        `http://localhost:${metricsPort}/metrics`,\n        changeStreamer,\n        // The time between when the zero-cache was started to when the\n        // change-streamer is ready to start serves as the initial delay for\n        // watermark cleanup (as it either includes a similar replica\n        // restoration/preparation step, or an initial-sync, which\n        // generally takes longer).\n        //\n        // Consider: Also account for permanent volumes?\n        Date.now() - parentStartMs,\n      )\n    : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    config,\n    {port, startupDelayMs},\n    parent,\n    changeStreamer,\n    monitor instanceof BackupMonitor ? monitor : null,\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer.\n  return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(() =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAmCA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;AACf,QAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,gBAAgB,SAAS,KAAK,GAAG;CAEvC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;CAClE,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,eACE;AAEJ,eAAc,iBAAiB,QAAQ,EAAC,QAAQ,mBAAkB,EAAE,MAAM,CAAC;CAC3E,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,mBAAkB,EAAE,KAAK;AACtE,eAAc,IAAI,OAAO;CAGzB,MAAM,WAAW,SACf,IACA,OAAO,IACP;EACE,KAAK,OAAO;EACZ,YAAY,GAAE,qBAAqB,0BAAyB;EAC7D,EACD,EAAC,kBAAkB,MAAK,CACzB;AACI,mBAAkB,IAAI,UAAU,SAAS,CAAC,YAAY,GAAG;CAE9D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,OAAO;CAEpC,IAAI;CAEJ,MAAM,UAAU,iBAAiB,OAAO;AAExC,MAAK,MAAM,SAAS,CAAC,MAAM,MAAM,CAC/B,KAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GACnC,EACD,SACA,eAAe,iBAChB,GACD,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,QACD;AAKP,mBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cATA,2BAA2B,eAAe,QAAQ,KAAK,EAWvD,mBACA,aAAa,OACb,iCACA,oCACA,WACD;AACD;UACO,GAAG;AACV,MAAI,SAAS,aAAa,iBAAiB;AACzC,MAAG,OAAO,qBAAqB,QAAQ,QAAQ,EAAE;AAGjD,gBAAa,QAAQ,KAAK;AAC1B;;AAEF,QAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,EAAE,CAC9C;AACD,MAAI,aAAa,kBACf,OAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,GAAE,CACX;AAEH,QAAM;;AAIV,QAAO,gBAAgB,mDAAmD;AAO1E,OAAM,eAAe,IAAI,wBAAwB,QAAQ,KAAK;CAE9D,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,QAAQ,MACR,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,KAAK,GAAG,cACd,GACD,IAAI,eAAe,IAAI,QAAQ,MAAM,eAAe;CAExD,MAAM,0BAA0B,IAAI,yBAClC,IACA,QACA;EAAC;EAAM;EAAe,EACtB,QACA,gBACA,mBAAmB,gBAAgB,UAAU,KAC9C;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAIrC,QAAO,eAAe,IAAI,QAAQ,yBAAyB,QAAQ;;AAIrE,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {\n  AutoResetSignal,\n  CHANGE_STREAMER_APP_NAME,\n} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  BackupNotFoundException,\n  restoreReplica,\n} from '../services/litestream/commands.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const workerStartTime = Date.now();\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusPaddingSeconds,\n    },\n    upstream,\n    change,\n    replica,\n    initialSync,\n    litestream,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  const lc = createLogContext(config, 'change-streamer');\n  initEventSink(lc, config);\n\n  // Kick off DB connection warmup in the background.\n  const changeDB = pgClient(\n    lc,\n    change.db,\n    {\n      max: change.maxConns,\n      connection: {['application_name']: CHANGE_STREAMER_APP_NAME},\n    },\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const {autoReset, replicationLag} = config;\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date, then acquire\n  // a lock to prevent change-lock purges. This ensures that (this)\n  // change-streamer will be able to resume from the backup.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();\n\n  // Restore from litestream if the change-log has entries.\n  if (purgeLock) {\n    try {\n      await restoreReplica(lc, config, purgeLock);\n    } catch (e) {\n      // If the restore failed, e.g. due to a corrupt or missing backup, the\n      // replication-manager recovers by re-syncing.\n      const log = e instanceof BackupNotFoundException ? 'warn' : 'error';\n      lc[log]?.(\n        `error restoring backup. resyncing the replica: ${String(e)}`,\n        e,\n      );\n\n      // The purgeLock must be released if the backup could not be restored,\n      // or it will otherwise prevent the change-db update after the resync\n      // completes.\n      await purgeLock.release();\n      purgeLock = null;\n    }\n  }\n\n  let changeStreamer: ChangeStreamerService | undefined;\n\n  const context = getServerContext(config);\n\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {changeSource, subscriptionState} =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        purgeLock,\n        autoReset ?? false,\n        {\n          backPressureLimitHeapProportion,\n          flowControlConsensusPaddingSeconds,\n          statementTimeoutMs: change.statementTimeoutMs,\n        },\n        setTimeout,\n      );\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      await publishCriticalEvent(\n        lc,\n        replicationStatusError(lc, 'Initializing', e),\n      );\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  // Perform any upgrades to the replica in case it was restored from an\n  // earlier version. Note that this upgrade is done by the replicator worker\n  // as well (in both the replication-manager and the view-syncer), but the\n  // change-streamer independently reads the replica, and it is fine run the\n  // upgrade logic redundantly since it is idempotent.\n  await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n  const {backupURL, port: metricsPort} = litestream;\n  const monitor = backupURL\n    ? new BackupMonitor(\n        lc,\n        replica.file,\n        backupURL,\n        `http://localhost:${metricsPort}/metrics`,\n        changeStreamer,\n        // The time between when the zero-cache was started to when the\n        // change-streamer is ready to start serves as the initial delay for\n        // watermark cleanup (as it either includes a similar replica\n        // restoration/preparation step, or an initial-sync, which\n        // generally takes longer).\n        //\n        // Consider: Also account for permanent volumes?\n        Date.now() - workerStartTime,\n      )\n    : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    config,\n    {port, startupDelayMs},\n    parent,\n    changeStreamer,\n    monitor instanceof BackupMonitor ? monitor : null,\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer.\n  return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(() =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,kBAAkB,KAAK,KAAK;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK;EAAK,CAAC;CACnD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,eACE;AAEJ,eACE,iBAAiB,QAAQ,mBAAmB,GAAG,MAAM,EACrD,mBACA,EACD;CACD,MAAM,KAAK,iBAAiB,QAAQ,kBAAkB;AACtD,eAAc,IAAI,OAAO;CAGzB,MAAM,WAAW,SACf,IACA,OAAO,IACP;EACE,KAAK,OAAO;EACZ,YAAY,GAAE,qBAAqB,0BAAyB;EAC7D,EACD,EAAC,kBAAkB,MAAK,CACzB;AACI,mBAAkB,IAAI,UAAU,SAAS,CAAC,YAAY,GAAG;CAE9D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,OAAO;AAKpC,OAAM,yBAAyB,IAAI,UAAU,MAAM;CACnD,IAAI,YAAY,MAAM,IAAI,YAAY,IAAI,OAAO,SAAS,CAAC,SAAS;AAGpE,KAAI,UACF,KAAI;AACF,QAAM,eAAe,IAAI,QAAQ,UAAU;UACpC,GAAG;AAIV,KADY,aAAa,0BAA0B,SAAS,WAE1D,kDAAkD,OAAO,EAAE,IAC3D,EACD;AAKD,QAAM,UAAU,SAAS;AACzB,cAAY;;CAIhB,IAAI;CAEJ,MAAM,UAAU,iBAAiB,OAAO;AAExC,MAAK,MAAM,SAAS,CAAC,MAAM,MAAM,CAC/B,KAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GACnC,EACD,SACA,eAAe,iBAChB,GACD,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,QACD;AAKP,mBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cATA,2BAA2B,eAAe,QAAQ,KAAK,EAWvD,mBACA,WACA,aAAa,OACb;GACE;GACA;GACA,oBAAoB,OAAO;GAC5B,EACD,WACD;AACD;UACO,GAAG;AACV,MAAI,SAAS,aAAa,iBAAiB;AACzC,MAAG,OAAO,qBAAqB,QAAQ,QAAQ,EAAE;AAGjD,gBAAa,QAAQ,KAAK;AAC1B;;AAEF,QAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,EAAE,CAC9C;AACD,MAAI,aAAa,kBACf,OAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,GAAE,CACX;AAEH,QAAM;;AAIV,QAAO,gBAAgB,mDAAmD;AAO1E,OAAM,eAAe,IAAI,wBAAwB,QAAQ,KAAK;CAE9D,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,QAAQ,MACR,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,KAAK,GAAG,gBACd,GACD,IAAI,eAAe,IAAI,QAAQ,MAAM,eAAe;CAExD,MAAM,0BAA0B,IAAI,yBAClC,IACA,QACA;EAAC;EAAM;EAAe,EACtB,QACA,gBACA,mBAAmB,gBAAgB,UAAU,KAC9C;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAIrC,QAAO,eAAe,IAAI,QAAQ,yBAAyB,QAAQ;;AAIrE,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}

package/out/zero-cache/src/server/logging.d.ts

@@ -2,7 +2,5 @@ import type { LogContext } from '@rocicorp/logger';
 import { type LogConfig } from '../../../shared/src/logging.ts';
 export declare function createLogContext({ log }: {
     log: LogConfig;
-}, context: {
-    worker: string;
-}, includeOtel?: boolean): LogContext;
+}, worker: string, workerIndex?: number, includeOtel?: boolean): LogContext;
 //# sourceMappingURL=logging.d.ts.map

package/out/zero-cache/src/server/logging.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"logging.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/logging.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,gCAAgC,CAAC;AAIxC,wBAAgB,gBAAgB,CAC9B,EAAC,GAAG,EAAC,EAAE;IAAC,GAAG,EAAE,SAAS,CAAA;CAAC,EACvB,OAAO,EAAE;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,EACzB,WAAW,UAAO,GACjB,UAAU,CASZ"}
+{"version":3,"file":"logging.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/logging.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,gCAAgC,CAAC;AAIxC,wBAAgB,gBAAgB,CAC9B,EAAC,GAAG,EAAC,EAAE;IAAC,GAAG,EAAE,SAAS,CAAA;CAAC,EACvB,MAAM,EAAE,MAAM,EACd,WAAW,SAAI,EACf,WAAW,UAAO,GACjB,UAAU,CASZ"}

package/out/zero-cache/src/server/logging.js

@@ -1,12 +1,15 @@
+import { createLogContext as createLogContext$1, getLogSink } from "../../../shared/src/logging.js";
 import "../services/life-cycle.js";
 import { otelLogsEnabled } from "../../../otel/src/enabled.js";
-import { createLogContext as createLogContext$1, getLogSink } from "../../../shared/src/logging.js";
 import { OtelLogSink } from "./otel-log-sink.js";
 import "@rocicorp/logger";
 //#region ../zero-cache/src/server/logging.ts
-function createLogContext({ log }, context, includeOtel = true) {
+function createLogContext({ log }, worker, workerIndex = 0, includeOtel = true) {
 	const logSink = createLogSink(log, includeOtel);
-	const lc = createLogContext$1({ log }, context, logSink);
+	const lc = createLogContext$1({ log }, {
+		worker,
+		workerIndex
+	}, logSink);
 	process.on("uncaughtException", async (err, origin) => {
 		lc.error?.(origin, err);
 		await logSink.flush?.();

package/out/zero-cache/src/server/logging.js.map

@@ -1 +1 @@
-{"version":3,"file":"logging.js","names":["#sinks"],"sources":["../../../../../zero-cache/src/server/logging.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Context, type LogLevel, type LogSink} from '@rocicorp/logger';\nimport {otelLogsEnabled} from '../../../otel/src/enabled.ts';\nimport {\n  createLogContext as createLogContextShared,\n  getLogSink,\n  type LogConfig,\n} from '../../../shared/src/logging.ts';\nimport {UNHANDLED_EXCEPTION_ERROR_CODE} from '../services/life-cycle.ts';\nimport {OtelLogSink} from './otel-log-sink.ts';\n\nexport function createLogContext(\n  {log}: {log: LogConfig},\n  context: {worker: string},\n  includeOtel = true,\n): LogContext {\n  const logSink = createLogSink(log, includeOtel);\n  const lc = createLogContextShared({log}, context, logSink);\n  process.on('uncaughtException', async (err, origin) => {\n    lc.error?.(origin, err);\n    await logSink.flush?.();\n    process.exit(UNHANDLED_EXCEPTION_ERROR_CODE);\n  });\n  return lc;\n}\n\nfunction createLogSink(config: LogConfig, includeOtel: boolean): LogSink {\n  const sink = getLogSink(config);\n  if (includeOtel && otelLogsEnabled()) {\n    const otelSink = new OtelLogSink();\n    return new CompositeLogSink([otelSink, sink]);\n  }\n  return sink;\n}\n\nclass CompositeLogSink implements LogSink {\n  readonly #sinks: LogSink[];\n\n  constructor(sinks: LogSink[]) {\n    this.#sinks = sinks;\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    for (const sink of this.#sinks) {\n      sink.log(level, context, ...args);\n    }\n  }\n}\n"],"mappings":";;;;;;AAWA,SAAgB,iBACd,EAAC,OACD,SACA,cAAc,MACF;CACZ,MAAM,UAAU,cAAc,KAAK,YAAY;CAC/C,MAAM,KAAK,mBAAuB,EAAC,KAAI,EAAE,SAAS,QAAQ;AAC1D,SAAQ,GAAG,qBAAqB,OAAO,KAAK,WAAW;AACrD,KAAG,QAAQ,QAAQ,IAAI;AACvB,QAAM,QAAQ,SAAS;AACvB,UAAQ,KAAA,GAAoC;GAC5C;AACF,QAAO;;AAGT,SAAS,cAAc,QAAmB,aAA+B;CACvE,MAAM,OAAO,WAAW,OAAO;AAC/B,KAAI,eAAe,iBAAiB,CAElC,QAAO,IAAI,iBAAiB,CADX,IAAI,aAAa,EACK,KAAK,CAAC;AAE/C,QAAO;;AAGT,IAAM,mBAAN,MAA0C;CACxC;CAEA,YAAY,OAAkB;AAC5B,QAAA,QAAc;;CAGhB,IAAI,OAAiB,SAA8B,GAAG,MAAuB;AAC3E,OAAK,MAAM,QAAQ,MAAA,MACjB,MAAK,IAAI,OAAO,SAAS,GAAG,KAAK"}
+{"version":3,"file":"logging.js","names":["#sinks"],"sources":["../../../../../zero-cache/src/server/logging.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Context, type LogLevel, type LogSink} from '@rocicorp/logger';\nimport {otelLogsEnabled} from '../../../otel/src/enabled.ts';\nimport {\n  createLogContext as createLogContextShared,\n  getLogSink,\n  type LogConfig,\n} from '../../../shared/src/logging.ts';\nimport {UNHANDLED_EXCEPTION_ERROR_CODE} from '../services/life-cycle.ts';\nimport {OtelLogSink} from './otel-log-sink.ts';\n\nexport function createLogContext(\n  {log}: {log: LogConfig},\n  worker: string,\n  workerIndex = 0,\n  includeOtel = true,\n): LogContext {\n  const logSink = createLogSink(log, includeOtel);\n  const lc = createLogContextShared({log}, {worker, workerIndex}, logSink);\n  process.on('uncaughtException', async (err, origin) => {\n    lc.error?.(origin, err);\n    await logSink.flush?.();\n    process.exit(UNHANDLED_EXCEPTION_ERROR_CODE);\n  });\n  return lc;\n}\n\nfunction createLogSink(config: LogConfig, includeOtel: boolean): LogSink {\n  const sink = getLogSink(config);\n  if (includeOtel && otelLogsEnabled()) {\n    const otelSink = new OtelLogSink();\n    return new CompositeLogSink([otelSink, sink]);\n  }\n  return sink;\n}\n\nclass CompositeLogSink implements LogSink {\n  readonly #sinks: LogSink[];\n\n  constructor(sinks: LogSink[]) {\n    this.#sinks = sinks;\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    for (const sink of this.#sinks) {\n      sink.log(level, context, ...args);\n    }\n  }\n}\n"],"mappings":";;;;;;AAWA,SAAgB,iBACd,EAAC,OACD,QACA,cAAc,GACd,cAAc,MACF;CACZ,MAAM,UAAU,cAAc,KAAK,YAAY;CAC/C,MAAM,KAAK,mBAAuB,EAAC,KAAI,EAAE;EAAC;EAAQ;EAAY,EAAE,QAAQ;AACxE,SAAQ,GAAG,qBAAqB,OAAO,KAAK,WAAW;AACrD,KAAG,QAAQ,QAAQ,IAAI;AACvB,QAAM,QAAQ,SAAS;AACvB,UAAQ,KAAA,GAAoC;GAC5C;AACF,QAAO;;AAGT,SAAS,cAAc,QAAmB,aAA+B;CACvE,MAAM,OAAO,WAAW,OAAO;AAC/B,KAAI,eAAe,iBAAiB,CAElC,QAAO,IAAI,iBAAiB,CADX,IAAI,aAAa,EACK,KAAK,CAAC;AAE/C,QAAO;;AAGT,IAAM,mBAAN,MAA0C;CACxC;CAEA,YAAY,OAAkB;AAC5B,QAAA,QAAc;;CAGhB,IAAI,OAAiB,SAA8B,GAAG,MAAuB;AAC3E,OAAK,MAAM,QAAQ,MAAA,MACjB,MAAK,IAAI,OAAO,SAAS,GAAG,KAAK"}

package/out/zero-cache/src/server/main.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAoB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CA+Kf"}
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAoB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAoKf"}

package/out/zero-cache/src/server/main.js

@@ -1,12 +1,12 @@
 import { must } from "../../../shared/src/must.js";
 import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
-import { ProcessManager, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
-import { startOtelAuto } from "./otel-start.js";
+import { ProcessManager, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { CHANGE_STREAMER_URL, REAPER_URL, REPLICATOR_URL, SYNCER_URL } from "./worker-urls.js";
 import { initEventSink } from "../observability/events.js";
 import { restoreReplica, startReplicaBackupProcess } from "../services/litestream/commands.js";
+import { startOtelAuto } from "./otel-start.js";
 import { createNotifierFrom, handleSubscriptionsFrom, subscribeTo } from "../workers/replicator.js";
 import { WorkerDispatcher } from "./worker-dispatcher.js";
 import { resolver } from "@rocicorp/resolver";
@@ -15,8 +15,8 @@ import path from "node:path";
 async function runWorker(parent, env) {
 	const startMs = Date.now();
 	const config = getNormalizedZeroConfig({ env });
-	startOtelAuto(createLogContext(config, { worker: "dispatcher" }, false));
-	const lc = createLogContext(config, { worker: "dispatcher" }, true);
+	startOtelAuto(createLogContext(config, "dispatcher", 0, false), "dispatcher", 0);
+	const lc = createLogContext(config, "dispatcher");
 	initEventSink(lc, config);
 	const processes = new ProcessManager(lc, parent);
 	const { numSyncWorkers: numSyncers } = config;
@@ -35,29 +35,29 @@ async function runWorker(parent, env) {
 	}
 	const { taskID, changeStreamer: { mode: changeStreamerMode, uri: changeStreamerURI }, litestream } = config;
 	const runChangeStreamer = changeStreamerMode === "dedicated" && changeStreamerURI === void 0;
-	let restoreStart = /* @__PURE__ */ new Date();
-	if (litestream.backupURL || litestream.executable && !runChangeStreamer) try {
-		restoreStart = await restoreReplica(lc, config);
-	} catch (e) {
-		if (runChangeStreamer) lc.error?.("error restoring backup. resyncing the replica.", e);
-		else throw e;
+	let changeStreamer;
+	if (!runChangeStreamer) {
+		changeStreamer = void 0;
+		if (litestream.executable) await restoreReplica(lc, config, null);
+	} else {
+		const { promise: changeStreamerReady, resolve: changeStreamerStarted } = resolver();
+		changeStreamer = loadWorker(CHANGE_STREAMER_URL, "supporting").once("message", changeStreamerStarted);
+		await changeStreamerReady;
+		if (litestream.backupURL) {
+			const { promise: backupReady, resolve } = resolver();
+			const mode = "backup";
+			loadWorker(REPLICATOR_URL, "supporting", mode, mode).once("message", () => {
+				processes.addSubprocess(startReplicaBackupProcess(lc, config), "supporting", "litestream");
+				resolve();
+			});
+			await backupReady;
+		}
 	}
-	const { promise: changeStreamerReady, resolve: changeStreamerStarted } = resolver();
-	const changeStreamer = runChangeStreamer ? loadWorker(CHANGE_STREAMER_URL, "supporting", void 0, String(restoreStart.getTime())).once("message", changeStreamerStarted) : changeStreamerStarted() ?? void 0;
-	const { promise: reaperReady, resolve: reaperStarted } = resolver();
-	if (numSyncers > 0) loadWorker(REAPER_URL, "supporting").once("message", reaperStarted);
-	else reaperStarted();
-	await changeStreamerReady;
-	if (runChangeStreamer && litestream.backupURL) {
-		const { promise: backupReady, resolve } = resolver();
-		const mode = "backup";
-		loadWorker(REPLICATOR_URL, "supporting", mode, mode).once("message", () => {
-			processes.addSubprocess(startReplicaBackupProcess(lc, config), "supporting", "litestream");
-			resolve();
-		});
-		await backupReady;
+	if (numSyncers > 0) {
+		const { promise: reaperReady, resolve: reaperStarted } = resolver();
+		loadWorker(REAPER_URL, "supporting").once("message", reaperStarted);
+		await reaperReady;
 	}
-	await reaperReady;
 	const syncers = [];
 	if (numSyncers) {
 		const mode = runChangeStreamer && litestream.backupURL ? "serving-copy" : "serving";
@@ -68,7 +68,7 @@ async function runWorker(parent, env) {
 		});
 		await replicaReady;
 		const notifier = createNotifierFrom(lc, replicator);
-		for (let i = 0; i < numSyncers; i++) syncers.push(loadWorker(SYNCER_URL, "user-facing", i + 1, mode));
+		for (let i = 0; i < numSyncers; i++) syncers.push(loadWorker(SYNCER_URL, "user-facing", i, mode, String(i)));
 		syncers.forEach((syncer) => handleSubscriptionsFrom(lc, syncer, notifier));
 	}
 	let mutator;

package/out/zero-cache/src/server/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import {resolver} from '@rocicorp/resolver';\nimport path from 'node:path';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  restoreReplica,\n  startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  type ReplicaFileMode,\n  subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(createLogContext(config, {worker: 'dispatcher'}, false));\n  const lc = createLogContext(config, {worker: 'dispatcher'}, true);\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    return processes.addWorker(worker, type, name);\n  }\n\n  const {\n    taskID,\n    changeStreamer: {mode: changeStreamerMode, uri: changeStreamerURI},\n    litestream,\n  } = config;\n  const runChangeStreamer =\n    changeStreamerMode === 'dedicated' && changeStreamerURI === undefined;\n\n  let restoreStart = new Date();\n  if (litestream.backupURL || (litestream.executable && !runChangeStreamer)) {\n    try {\n      restoreStart = await restoreReplica(lc, config);\n    } catch (e) {\n      if (runChangeStreamer) {\n        // If the restore failed, e.g. due to a corrupt backup, the\n        // replication-manager recovers by re-syncing.\n        lc.error?.('error restoring backup. resyncing the replica.', e);\n      } else {\n        // View-syncers, on the other hand, have no option other than to retry\n        // until a valid backup has been published. This is achieved by\n        // shutting down and letting the container runner retry with its\n        // configured policy.\n        throw e;\n      }\n    }\n  }\n\n  const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n    resolver();\n  const changeStreamer = runChangeStreamer\n    ? loadWorker(\n        CHANGE_STREAMER_URL,\n        'supporting',\n        undefined,\n        String(restoreStart.getTime()),\n      ).once('message', changeStreamerStarted)\n    : (changeStreamerStarted() ?? undefined);\n\n  const {promise: reaperReady, resolve: reaperStarted} = resolver();\n  if (numSyncers > 0) {\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n  } else {\n    reaperStarted();\n  }\n\n  // Wait for the change-streamer to be ready to guarantee that a replica\n  // file is present.\n  await changeStreamerReady;\n\n  if (runChangeStreamer && litestream.backupURL) {\n    // Start a backup replicator and corresponding litestream backup process.\n    const {promise: backupReady, resolve} = resolver();\n    const mode: ReplicaFileMode = 'backup';\n    loadWorker(REPLICATOR_URL, 'supporting', mode, mode).once(\n      // Wait for the Replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the lock when the\n      // replicator first prepares the db file.\n      'message',\n      () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, config),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      },\n    );\n    await backupReady;\n  }\n\n  // Before starting the view-syncers, ensure that the reaper has started\n  // up, indicating that any CVR db migrations have been performed.\n  await reaperReady;\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i + 1, mode));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  lc.info?.(`all workers ready (${Date.now() - startMs} ms)`);\n\n  parent.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  }\n\n  await processes.done();\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(() => runWorker(must(parentWorker), process.env));\n}\n"],"mappings":";;;;;;;;;;;;;;AAwCA,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,KAAK;CAC1B,MAAM,SAAS,wBAAwB,EAAC,KAAI,CAAC;AAE7C,eAAc,iBAAiB,QAAQ,EAAC,QAAQ,cAAa,EAAE,MAAM,CAAC;CACtE,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,cAAa,EAAE,KAAK;AACjE,eAAc,IAAI,OAAO;CAEzB,MAAM,YAAY,IAAI,eAAe,IAAI,OAAO;CAEhD,MAAM,EAAC,gBAAgB,eAAc;AACrC,KAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,WAC3D,OAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,kHAEnF;AAEH,KAAI,OAAO,IAAI,WAAW,WACxB,OAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,6GAEzE;CAGH,MAAM,gBACJ,eAAe,IACX,EAAE,GACF;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,WAAW,CAAC;EACzD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,WAAW,CAAC;EACrD;CAEP,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,cAAc;EACrE,MAAM,OAAO,KAAK,SAAS,UAAU,SAAS,IAAI,KAAK,KAAK,GAAG,KAAK;AACpE,SAAO,UAAU,UAAU,QAAQ,MAAM,KAAK;;CAGhD,MAAM,EACJ,QACA,gBAAgB,EAAC,MAAM,oBAAoB,KAAK,qBAChD,eACE;CACJ,MAAM,oBACJ,uBAAuB,eAAe,sBAAsB,KAAA;CAE9D,IAAI,+BAAe,IAAI,MAAM;AAC7B,KAAI,WAAW,aAAc,WAAW,cAAc,CAAC,kBACrD,KAAI;AACF,iBAAe,MAAM,eAAe,IAAI,OAAO;UACxC,GAAG;AACV,MAAI,kBAGF,IAAG,QAAQ,kDAAkD,EAAE;MAM/D,OAAM;;CAKZ,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,UAAU;CACZ,MAAM,iBAAiB,oBACnB,WACE,qBACA,cACA,KAAA,GACA,OAAO,aAAa,SAAS,CAAC,CAC/B,CAAC,KAAK,WAAW,sBAAsB,GACvC,uBAAuB,IAAI,KAAA;CAEhC,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,UAAU;AACjE,KAAI,aAAa,EACf,YAAW,YAAY,aAAa,CAAC,KAAK,WAAW,cAAc;KAEnE,gBAAe;AAKjB,OAAM;AAEN,KAAI,qBAAqB,WAAW,WAAW;EAE7C,MAAM,EAAC,SAAS,aAAa,YAAW,UAAU;EAClD,MAAM,OAAwB;AAC9B,aAAW,gBAAgB,cAAc,MAAM,KAAK,CAAC,KAInD,iBACM;AACJ,aAAU,cACR,0BAA0B,IAAI,OAAO,EACrC,cACA,aACD;AACD,YAAS;IAEZ;AACD,QAAM;;AAKR,OAAM;CAEN,MAAM,UAAoB,EAAE;AAC5B,KAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,UAAU;EACnD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,KACD,CAAC,KAAK,iBAAiB;AACtB,eAAY,IAAI,WAAW;AAC3B,YAAS;IACT;AACF,QAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,WAAW;AACnD,OAAK,IAAI,IAAI,GAAG,IAAI,YAAY,IAC9B,SAAQ,KAAK,WAAW,YAAY,eAAe,IAAI,GAAG,KAAK,CAAC;AAElE,UAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,SAAS,CAAC;;CAE1E,IAAI;AAKJ,IAAG,OAAO,sCAAsC;CAChD,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,cAAc,CAAC,KAAK,KAAK,GAAG,EAC3E,IACD;AACD,OAAM,UAAU,iBAAiB;AACjC,eAAc,WAAW;AACzB,IAAG,OAAO,sBAAsB,KAAK,KAAK,GAAG,QAAQ,MAAM;AAE3D,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAErC,KAAI;AACF,QAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,eACD,CACF;UACM,KAAK;AACZ,YAAU,gBAAgB,KAAK,aAAa;;AAG9C,OAAM,UAAU,MAAM;;AAGxB,IAAI,CAAC,mBAAmB,CACjB,iBAAgB,UAAU,KAAK,aAAa,EAAE,QAAQ,IAAI,CAAC"}
+{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  restoreReplica,\n  startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  type ReplicaFileMode,\n  subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  const lc = createLogContext(config, 'dispatcher');\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    return processes.addWorker(worker, type, name);\n  }\n\n  const {\n    taskID,\n    changeStreamer: {mode: changeStreamerMode, uri: changeStreamerURI},\n    litestream,\n  } = config;\n  const runChangeStreamer =\n    changeStreamerMode === 'dedicated' && changeStreamerURI === undefined;\n\n  let changeStreamer: Worker | undefined;\n\n  if (!runChangeStreamer) {\n    changeStreamer = undefined;\n    if (litestream.executable) {\n      // For view-syncers, the backup is restored here. For the replication-manager,\n      // the backup is restored in the change-streamer worker.\n      await restoreReplica(lc, config, null);\n    }\n  } else {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n      'message',\n      changeStreamerStarted,\n    );\n\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n\n    if (litestream.backupURL) {\n      // Start a backup replicator and corresponding litestream backup process.\n      const {promise: backupReady, resolve} = resolver();\n      const mode: ReplicaFileMode = 'backup';\n      loadWorker(REPLICATOR_URL, 'supporting', mode, mode).once(\n        // Wait for the Replicator's first message (i.e. \"ready\") before starting\n        // litestream backup in order to avoid contending on the lock when the\n        // replicator first prepares the db file.\n        'message',\n        () => {\n          processes.addSubprocess(\n            startReplicaBackupProcess(lc, config),\n            'supporting',\n            'litestream',\n          );\n          resolve();\n        },\n      );\n      await backupReady;\n    }\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  lc.info?.(`all workers ready (${Date.now() - startMs} ms)`);\n\n  parent.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  }\n\n  await processes.done();\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(() => runWorker(must(parentWorker), process.env));\n}\n"],"mappings":";;;;;;;;;;;;;;AAwCA,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,KAAK;CAC1B,MAAM,SAAS,wBAAwB,EAAC,KAAI,CAAC;AAE7C,eACE,iBAAiB,QAAQ,cAAc,GAAG,MAAM,EAChD,cACA,EACD;CACD,MAAM,KAAK,iBAAiB,QAAQ,aAAa;AACjD,eAAc,IAAI,OAAO;CAEzB,MAAM,YAAY,IAAI,eAAe,IAAI,OAAO;CAEhD,MAAM,EAAC,gBAAgB,eAAc;AACrC,KAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,WAC3D,OAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,kHAEnF;AAEH,KAAI,OAAO,IAAI,WAAW,WACxB,OAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,6GAEzE;CAGH,MAAM,gBACJ,eAAe,IACX,EAAE,GACF;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,WAAW,CAAC;EACzD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,WAAW,CAAC;EACrD;CAEP,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,cAAc;EACrE,MAAM,OAAO,KAAK,SAAS,UAAU,SAAS,IAAI,KAAK,KAAK,GAAG,KAAK;AACpE,SAAO,UAAU,UAAU,QAAQ,MAAM,KAAK;;CAGhD,MAAM,EACJ,QACA,gBAAgB,EAAC,MAAM,oBAAoB,KAAK,qBAChD,eACE;CACJ,MAAM,oBACJ,uBAAuB,eAAe,sBAAsB,KAAA;CAE9D,IAAI;AAEJ,KAAI,CAAC,mBAAmB;AACtB,mBAAiB,KAAA;AACjB,MAAI,WAAW,WAGb,OAAM,eAAe,IAAI,QAAQ,KAAK;QAEnC;EACL,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,UAAU;AACZ,mBAAiB,WAAW,qBAAqB,aAAa,CAAC,KAC7D,WACA,sBACD;AAID,QAAM;AAEN,MAAI,WAAW,WAAW;GAExB,MAAM,EAAC,SAAS,aAAa,YAAW,UAAU;GAClD,MAAM,OAAwB;AAC9B,cAAW,gBAAgB,cAAc,MAAM,KAAK,CAAC,KAInD,iBACM;AACJ,cAAU,cACR,0BAA0B,IAAI,OAAO,EACrC,cACA,aACD;AACD,aAAS;KAEZ;AACD,SAAM;;;AAIV,KAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,UAAU;AACjE,aAAW,YAAY,aAAa,CAAC,KAAK,WAAW,cAAc;AAGnE,QAAM;;CAGR,MAAM,UAAoB,EAAE;AAC5B,KAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,UAAU;EACnD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,KACD,CAAC,KAAK,iBAAiB;AACtB,eAAY,IAAI,WAAW;AAC3B,YAAS;IACT;AACF,QAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,WAAW;AACnD,OAAK,IAAI,IAAI,GAAG,IAAI,YAAY,IAC9B,SAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,EAAE,CAAC,CAAC;AAEzE,UAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,SAAS,CAAC;;CAE1E,IAAI;AAKJ,IAAG,OAAO,sCAAsC;CAChD,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,cAAc,CAAC,KAAK,KAAK,GAAG,EAC3E,IACD;AACD,OAAM,UAAU,iBAAiB;AACjC,eAAc,WAAW;AACzB,IAAG,OAAO,sBAAsB,KAAK,KAAK,GAAG,QAAQ,MAAM;AAE3D,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAErC,KAAI;AACF,QAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,eACD,CACF;UACM,KAAK;AACZ,YAAU,gBAAgB,KAAK,aAAa;;AAG9C,OAAM,UAAU,MAAM;;AAGxB,IAAI,CAAC,mBAAmB,CACjB,iBAAgB,UAAU,KAAK,aAAa,EAAE,QAAQ,IAAI,CAAC"}

package/out/zero-cache/src/server/mutator.js

@@ -1,9 +1,10 @@
 import { must } from "../../../shared/src/must.js";
 import { parentWorker, singleProcessMode } from "../types/processes.js";
-import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
+import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { initEventSink } from "../observability/events.js";
+import { startOtelAuto } from "./otel-start.js";
 import { Mutator } from "../workers/mutator.js";
 //#region ../zero-cache/src/server/mutator.ts
 function runWorker(parent, env, ...args) {
@@ -11,7 +12,8 @@ function runWorker(parent, env, ...args) {
 		env,
 		argv: args.slice(1)
 	});
-	const lc = createLogContext(config, { worker: "mutator" });
+	startOtelAuto(createLogContext(config, "mutator", 0, false), "mutator", 0);
+	const lc = createLogContext(config, "mutator");
 	initEventSink(lc, config);
 	return runUntilKilled(lc, parent, new Mutator());
 }

package/out/zero-cache/src/server/mutator.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\n\nfunction runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const lc = createLogContext(config, {worker: 'mutator'});\n  initEventSink(lc, config);\n\n  // TODO: create `PusherFactory`\n  return runUntilKilled(lc, parent, new Mutator());\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(() =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;AAYA,SAAS,UACP,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;CAClE,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,WAAU,CAAC;AACxD,eAAc,IAAI,OAAO;AAGzB,QAAO,eAAe,IAAI,QAAQ,IAAI,SAAS,CAAC;;AAGlD,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
+{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nfunction runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  startOtelAuto(createLogContext(config, 'mutator', 0, false), 'mutator', 0);\n  const lc = createLogContext(config, 'mutator');\n  initEventSink(lc, config);\n\n  // TODO: create `PusherFactory`\n  return runUntilKilled(lc, parent, new Mutator());\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(() =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;AAaA,SAAS,UACP,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;AAClE,eAAc,iBAAiB,QAAQ,WAAW,GAAG,MAAM,EAAE,WAAW,EAAE;CAC1E,MAAM,KAAK,iBAAiB,QAAQ,UAAU;AAC9C,eAAc,IAAI,OAAO;AAGzB,QAAO,eAAe,IAAI,QAAQ,IAAI,SAAS,CAAC;;AAGlD,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}

package/out/zero-cache/src/server/otel-log-sink.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"otel-log-sink.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/otel-log-sink.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAC,MAAM,kBAAkB,CAAC;AAKjE,qBAAa,WAAY,YAAW,OAAO;;;IAUzC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,SAAS,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;CAwB7E"}
+{"version":3,"file":"otel-log-sink.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/otel-log-sink.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAC,MAAM,kBAAkB,CAAC;AAIjE,qBAAa,WAAY,YAAW,OAAO;;;IAOzC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,SAAS,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;CAwB7E"}

package/out/zero-cache/src/server/otel-log-sink.js

@@ -1,12 +1,10 @@
 import { stringify } from "../../../shared/src/bigint-json.js";
 import { errorOrObject } from "../../../shared/src/logging.js";
-import { startOtelAuto } from "./otel-start.js";
 import { SeverityNumber, logs } from "@opentelemetry/api-logs";
 //#region ../zero-cache/src/server/otel-log-sink.ts
 var OtelLogSink = class {
 	#logger;
 	constructor() {
-		startOtelAuto();
 		this.#logger = logs.getLogger("zero-cache");
 	}
 	log(level, context, ...args) {

package/out/zero-cache/src/server/otel-log-sink.js.map

@@ -1 +1 @@
-{"version":3,"file":"otel-log-sink.js","names":["#logger"],"sources":["../../../../../zero-cache/src/server/otel-log-sink.ts"],"sourcesContent":["import {\n  logs,\n  SeverityNumber,\n  type AnyValueMap,\n  type Logger,\n  type LogRecord,\n} from '@opentelemetry/api-logs';\nimport type {Context, LogLevel, LogSink} from '@rocicorp/logger';\nimport {errorOrObject} from '../../../shared/src/logging.ts';\nimport {stringify} from '../../../shared/src/bigint-json.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport class OtelLogSink implements LogSink {\n  readonly #logger: Logger;\n\n  constructor() {\n    // start otel in case it was not started yet\n    // this is a no-op if already started\n    startOtelAuto();\n    this.#logger = logs.getLogger('zero-cache');\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    const lastObj = errorOrObject(args.at(-1));\n    if (lastObj) {\n      args.pop();\n    }\n\n    let message = args.length\n      ? args.map(s => (typeof s === 'string' ? s : stringify(s))).join(' ')\n      : '';\n\n    if (lastObj) {\n      message += ` ${stringify(lastObj)}`;\n    }\n\n    const payload: LogRecord = {\n      severityText: level,\n      severityNumber: toErrorNum(level),\n      body: message,\n    };\n    if (context) {\n      payload.attributes = context as AnyValueMap;\n    }\n    this.#logger.emit(payload);\n  }\n}\n\nfunction toErrorNum(level: LogLevel): SeverityNumber {\n  switch (level) {\n    case 'error':\n      return SeverityNumber.ERROR;\n    case 'warn':\n      return SeverityNumber.WARN;\n    case 'info':\n      return SeverityNumber.INFO;\n    case 'debug':\n      return SeverityNumber.DEBUG;\n    default:\n      throw new Error(`Unknown log level: ${level}`);\n  }\n}\n"],"mappings":";;;;;AAYA,IAAa,cAAb,MAA4C;CAC1C;CAEA,cAAc;AAGZ,iBAAe;AACf,QAAA,SAAe,KAAK,UAAU,aAAa;;CAG7C,IAAI,OAAiB,SAA8B,GAAG,MAAuB;EAC3E,MAAM,UAAU,cAAc,KAAK,GAAG,GAAG,CAAC;AAC1C,MAAI,QACF,MAAK,KAAK;EAGZ,IAAI,UAAU,KAAK,SACf,KAAK,KAAI,MAAM,OAAO,MAAM,WAAW,IAAI,UAAU,EAAE,CAAE,CAAC,KAAK,IAAI,GACnE;AAEJ,MAAI,QACF,YAAW,IAAI,UAAU,QAAQ;EAGnC,MAAM,UAAqB;GACzB,cAAc;GACd,gBAAgB,WAAW,MAAM;GACjC,MAAM;GACP;AACD,MAAI,QACF,SAAQ,aAAa;AAEvB,QAAA,OAAa,KAAK,QAAQ;;;AAI9B,SAAS,WAAW,OAAiC;AACnD,SAAQ,OAAR;EACE,KAAK,QACH,QAAO,eAAe;EACxB,KAAK,OACH,QAAO,eAAe;EACxB,KAAK,OACH,QAAO,eAAe;EACxB,KAAK,QACH,QAAO,eAAe;EACxB,QACE,OAAM,IAAI,MAAM,sBAAsB,QAAQ"}
+{"version":3,"file":"otel-log-sink.js","names":["#logger"],"sources":["../../../../../zero-cache/src/server/otel-log-sink.ts"],"sourcesContent":["import {\n  logs,\n  SeverityNumber,\n  type AnyValueMap,\n  type Logger,\n  type LogRecord,\n} from '@opentelemetry/api-logs';\nimport type {Context, LogLevel, LogSink} from '@rocicorp/logger';\nimport {stringify} from '../../../shared/src/bigint-json.ts';\nimport {errorOrObject} from '../../../shared/src/logging.ts';\n\nexport class OtelLogSink implements LogSink {\n  readonly #logger: Logger;\n\n  constructor() {\n    this.#logger = logs.getLogger('zero-cache');\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    const lastObj = errorOrObject(args.at(-1));\n    if (lastObj) {\n      args.pop();\n    }\n\n    let message = args.length\n      ? args.map(s => (typeof s === 'string' ? s : stringify(s))).join(' ')\n      : '';\n\n    if (lastObj) {\n      message += ` ${stringify(lastObj)}`;\n    }\n\n    const payload: LogRecord = {\n      severityText: level,\n      severityNumber: toErrorNum(level),\n      body: message,\n    };\n    if (context) {\n      payload.attributes = context as AnyValueMap;\n    }\n    this.#logger.emit(payload);\n  }\n}\n\nfunction toErrorNum(level: LogLevel): SeverityNumber {\n  switch (level) {\n    case 'error':\n      return SeverityNumber.ERROR;\n    case 'warn':\n      return SeverityNumber.WARN;\n    case 'info':\n      return SeverityNumber.INFO;\n    case 'debug':\n      return SeverityNumber.DEBUG;\n    default:\n      throw new Error(`Unknown log level: ${level}`);\n  }\n}\n"],"mappings":";;;;AAWA,IAAa,cAAb,MAA4C;CAC1C;CAEA,cAAc;AACZ,QAAA,SAAe,KAAK,UAAU,aAAa;;CAG7C,IAAI,OAAiB,SAA8B,GAAG,MAAuB;EAC3E,MAAM,UAAU,cAAc,KAAK,GAAG,GAAG,CAAC;AAC1C,MAAI,QACF,MAAK,KAAK;EAGZ,IAAI,UAAU,KAAK,SACf,KAAK,KAAI,MAAM,OAAO,MAAM,WAAW,IAAI,UAAU,EAAE,CAAE,CAAC,KAAK,IAAI,GACnE;AAEJ,MAAI,QACF,YAAW,IAAI,UAAU,QAAQ;EAGnC,MAAM,UAAqB;GACzB,cAAc;GACd,gBAAgB,WAAW,MAAM;GACjC,MAAM;GACP;AACD,MAAI,QACF,SAAQ,aAAa;AAEvB,QAAA,OAAa,KAAK,QAAQ;;;AAI9B,SAAS,WAAW,OAAiC;AACnD,SAAQ,OAAR;EACE,KAAK,QACH,QAAO,eAAe;EACxB,KAAK,OACH,QAAO,eAAe;EACxB,KAAK,OACH,QAAO,eAAe;EACxB,KAAK,QACH,QAAO,eAAe;EACxB,QACE,OAAM,IAAI,MAAM,sBAAsB,QAAQ"}

package/out/zero-cache/src/server/otel-start.d.ts

@@ -1,3 +1,3 @@
 import type { LogContext } from '@rocicorp/logger';
-export declare const startOtelAuto: (lc?: LogContext) => void;
+export declare const startOtelAuto: (lc: LogContext | undefined, workerName: string, workerIndex: number) => void;
 //# sourceMappingURL=otel-start.d.ts.map

package/out/zero-cache/src/server/otel-start.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"otel-start.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/otel-start.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AA6EjD,eAAO,MAAM,aAAa,GAAI,KAAK,UAAU,SACA,CAAC"}
+{"version":3,"file":"otel-start.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/otel-start.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AA0FjD,eAAO,MAAM,aAAa,GACxB,IAAI,UAAU,GAAG,SAAS,EAC1B,YAAY,MAAM,EAClB,aAAa,MAAM,SACoD,CAAC"}

package/out/zero-cache/src/server/otel-start.js

@@ -14,13 +14,17 @@ var OtelManager = class OtelManager {
 		if (!OtelManager.#instance) OtelManager.#instance = new OtelManager();
 		return OtelManager.#instance;
 	}
-	startOtelAuto(lc) {
+	startOtelAuto(lc, workerName, workerIndex) {
 		if (this.#started || !otelEnabled()) return;
 		this.#started = true;
 		const otelLogLevel = process.env.OTEL_LOG_LEVEL;
 		delete process.env.OTEL_LOG_LEVEL;
 		process.env.OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION ??= "base2_exponential_bucket_histogram";
-		const resource = resourceFromAttributes({ [ATTR_SERVICE_VERSION]: process.env.ZERO_SERVER_VERSION ?? "unknown" });
+		const resource = resourceFromAttributes({
+			[ATTR_SERVICE_VERSION]: process.env.ZERO_SERVER_VERSION ?? "unknown",
+			"process.worker": workerName,
+			"process.worker_index": workerIndex
+		});
 		process.env.OTEL_EXPORTER_OTLP_PROTOCOL ??= "http/json";
 		process.env.OTEL_METRICS_EXPORTER ??= otelMetricsEnabled() ? "otlp" : "none";
 		process.env.OTEL_TRACES_EXPORTER ??= otelTracesEnabled() ? "otlp" : "none";
@@ -42,7 +46,7 @@ var OtelManager = class OtelManager {
 		});
 	}
 };
-var startOtelAuto = (lc) => OtelManager.getInstance().startOtelAuto(lc);
+var startOtelAuto = (lc, workerName, workerIndex) => OtelManager.getInstance().startOtelAuto(lc, workerName, workerIndex);
 //#endregion
 export { startOtelAuto };