@rocicorp/zero 1.7.0-canary.5 → 1.8.0-canary.0

package/out/zero-cache/src/custom/fetch.js

@@ -30,19 +30,6 @@ function compileUrlPattern(pattern) {
 		throw new Error(`Invalid URLPattern in URL configuration: "${pattern}". Error: ${e instanceof Error ? e.message : String(e)}`);
 	}
 }
-/**
-* Filters custom headers based on the allowed headers list.
-* If no allowedHeaders list is specified, no custom headers are forwarded (secure by default).
-* Header names are compared case-insensitively.
-*/
-function filterCustomHeaders(headers, allowedHeaders) {
-	if (!headers) return {};
-	if (!allowedHeaders || allowedHeaders.length === 0) return {};
-	const allowed = new Set(allowedHeaders.map((h) => h.toLowerCase()));
-	const filtered = {};
-	for (const [key, value] of Object.entries(headers)) if (allowed.has(key.toLowerCase())) filtered[key] = value;
-	return filtered;
-}
 var getBodyPreview = async (res, lc) => {
 	try {
 		const body = await res.clone().text();
@@ -75,7 +62,7 @@ async function fetchFromAPIServer(validator, source, lc, ctx, shard, body) {
 	}, "warn");
 	const headers = { "Content-Type": "application/json" };
 	if (headerOptions.apiKey) headers["X-Api-Key"] = headerOptions.apiKey;
-	Object.assign(headers, filterCustomHeaders(headerOptions.customHeaders, headerOptions.allowedClientHeaders));
+	Object.assign(headers, headerOptions.customHeaders, headerOptions.requestHeaders);
 	if (ctx.auth?.raw) headers["Authorization"] = `Bearer ${ctx.auth.raw}`;
 	if (headerOptions.cookie) headers["Cookie"] = headerOptions.cookie;
 	if (headerOptions.origin) headers["Origin"] = headerOptions.origin;

package/out/zero-cache/src/custom/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","names":[],"sources":["../../../../../zero-cache/src/custom/fetch.ts"],"sourcesContent":["import {context, propagation} from '@opentelemetry/api';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport 'urlpattern-polyfill';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../shared/src/error.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport {type Type} from '../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../zero-protocol/src/error-reason.ts';\nimport {isProtocolError} from '../../../zero-protocol/src/error.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {upstreamSchema, type ShardID} from '../types/shards.ts';\n\nconst reservedParams = ['schema', 'appID'];\n\n/**\n * Compiles and validates a URLPattern from configuration.\n *\n * Patterns must be full URLs (e.g., \"https://api.example.com/endpoint\").\n * URLPattern automatically sets search and hash to wildcard ('*'),\n * which means query parameters and fragments are ignored during matching.\n *\n * @throws Error if the pattern is an invalid URLPattern\n */\nexport function compileUrlPattern(pattern: string): URLPattern {\n  try {\n    return new URLPattern(pattern);\n  } catch (e) {\n    throw new Error(\n      `Invalid URLPattern in URL configuration: \"${pattern}\". Error: ${e instanceof Error ? e.message : String(e)}`,\n    );\n  }\n}\n\n/**\n * Filters custom headers based on the allowed headers list.\n * If no allowedHeaders list is specified, no custom headers are forwarded (secure by default).\n * Header names are compared case-insensitively.\n */\nfunction filterCustomHeaders(\n  headers: Readonly<Record<string, string>> | undefined,\n  allowedHeaders: readonly string[] | undefined,\n): Record<string, string> {\n  if (!headers) {\n    return {};\n  }\n  if (!allowedHeaders || allowedHeaders.length === 0) {\n    return {};\n  }\n\n  const allowed = new Set(allowedHeaders.map(h => h.toLowerCase()));\n  const filtered: Record<string, string> = {};\n  for (const [key, value] of Object.entries(headers)) {\n    if (allowed.has(key.toLowerCase())) {\n      filtered[key] = value;\n    }\n  }\n  return filtered;\n}\n\nexport const getBodyPreview = async (\n  res: Response,\n  lc: LogContext,\n): Promise<string | undefined> => {\n  try {\n    const body = await res.clone().text();\n    if (body.length > 512) {\n      return body.slice(0, 512) + '...';\n    }\n    return body;\n  } catch (e) {\n    lc.warn?.(\n      'failed to get body preview',\n      {\n        url: res.url,\n      },\n      e,\n    );\n  }\n\n  return undefined;\n};\n\nconst MAX_ATTEMPTS = 4;\n\nexport async function fetchFromAPIServer<TValidator extends Type>(\n  validator: TValidator,\n  source: 'push' | 'transform',\n  lc: LogContext,\n  ctx: ConnectionContext,\n  shard: ShardID,\n  body: ReadonlyJSONValue,\n) {\n  const fetchFromAPIServerID = randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n  lc = lc\n    .withContext('fetchFromAPIServerID', fetchFromAPIServerID)\n    .withContext('source', source);\n\n  const fetchConfig = source === 'push' ? ctx.mutateContext : ctx.queryContext;\n  const url = must(\n    fetchConfig.url,\n    `Fetch config for ${source} is missing URL`,\n  );\n  const headerOptions = fetchConfig.headerOptions;\n\n  lc.debug?.('fetchFromAPIServer called', {\n    url,\n  });\n\n  if (!urlMatch(url, fetchConfig.allowedUrlPatterns ?? [])) {\n    throw new ProtocolErrorWithLevel(\n      source === 'push'\n        ? {\n            kind: ErrorKind.PushFailed,\n            origin: ErrorOrigin.ZeroCache,\n            reason: ErrorReason.Internal,\n            message: `URL \"${url}\" is not allowed by the ZERO_MUTATE_URL configuration`,\n            mutationIDs: [],\n          }\n        : {\n            kind: ErrorKind.TransformFailed,\n            origin: ErrorOrigin.ZeroCache,\n            reason: ErrorReason.Internal,\n            message: `URL \"${url}\" is not allowed by the ZERO_QUERY_URL configuration`,\n            queryIDs: [],\n          },\n      'warn',\n    );\n  }\n  const headers: Record<string, string> = {\n    'Content-Type': 'application/json',\n  };\n\n  if (headerOptions.apiKey) {\n    headers['X-Api-Key'] = headerOptions.apiKey;\n  }\n  Object.assign(\n    headers,\n    filterCustomHeaders(\n      headerOptions.customHeaders,\n      headerOptions.allowedClientHeaders,\n    ),\n  );\n  if (ctx.auth?.raw) {\n    headers['Authorization'] = `Bearer ${ctx.auth.raw}`;\n  }\n  if (headerOptions.cookie) {\n    headers['Cookie'] = headerOptions.cookie;\n  }\n  if (headerOptions.origin) {\n    headers['Origin'] = headerOptions.origin;\n  }\n  propagation.inject(context.active(), headers);\n\n  const urlObj = new URL(url);\n  const params = new URLSearchParams(urlObj.search);\n\n  for (const reserved of reservedParams) {\n    assert(\n      !params.has(reserved),\n      `The push URL cannot contain the reserved query param \"${reserved}\"`,\n    );\n  }\n\n  params.append('schema', upstreamSchema(shard));\n  params.append('appID', shard.appID);\n\n  urlObj.search = params.toString();\n\n  const finalUrl = urlObj.toString();\n\n  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {\n    lc = lc.withContext('fetchFromAPIServerAttempt', attempt);\n    lc.debug?.('fetch from API server attempt');\n    const shouldRetry = async () => {\n      if (attempt < MAX_ATTEMPTS) {\n        const delayMs = getBackoffDelayMs(attempt);\n        lc.debug?.(`fetch from API server retrying in ${delayMs} ms`);\n        await sleep(delayMs);\n        return true;\n      }\n      lc.debug?.('fetch from API server reached max attempts, not retrying');\n      return false;\n    };\n    try {\n      const response = await fetch(finalUrl, {\n        method: 'POST',\n        headers,\n        body: JSON.stringify(body),\n      });\n\n      if (!response.ok) {\n        const bodyPreview = await getBodyPreview(response, lc);\n        lc.warn?.('fetch from API server returned non-OK status', {\n          url: finalUrl,\n          status: response.status,\n          bodyPreview,\n        });\n        // Bad Gateway or Gateway Timeout indicate the server was not reached\n        // We retry these if we have retries remaining.\n        if (\n          (response.status === 502 || response.status === 504) &&\n          (await shouldRetry())\n        ) {\n          continue;\n        }\n\n        throw new ProtocolErrorWithLevel(\n          source === 'push'\n            ? {\n                kind: ErrorKind.PushFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.HTTP,\n                status: response.status,\n                bodyPreview,\n                message: `Fetch from API server returned non-OK status ${response.status}`,\n                mutationIDs: [],\n              }\n            : {\n                kind: ErrorKind.TransformFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.HTTP,\n                status: response.status,\n                bodyPreview,\n                message: `Fetch from API server returned non-OK status ${response.status}`,\n                queryIDs: [],\n              },\n          'warn',\n        );\n      }\n\n      try {\n        const json = await response.json();\n        const result = validator.parse(json, {\n          mode: 'passthrough',\n        });\n        lc.debug?.('fetch from API server succeeded');\n        return result;\n      } catch (error) {\n        lc.warn?.(\n          'failed to parse response',\n          {\n            url: finalUrl,\n          },\n          error,\n        );\n\n        throw new ProtocolErrorWithLevel(\n          source === 'push'\n            ? {\n                kind: ErrorKind.PushFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.Parse,\n                message: `Failed to parse response from API server: ${getErrorMessage(error)}`,\n                mutationIDs: [],\n              }\n            : {\n                kind: ErrorKind.TransformFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.Parse,\n                message: `Failed to parse response from API server: ${getErrorMessage(error)}`,\n                queryIDs: [],\n              },\n          'warn',\n          {cause: error},\n        );\n      }\n    } catch (error) {\n      if (isProtocolError(error)) {\n        throw error;\n      }\n\n      const isFetchFailed =\n        error instanceof TypeError && error.message === 'fetch failed';\n      // unexpected/unknown errors should be logged at 'error' level so they\n      // are investigated\n      let logLevel: LogLevel = isFetchFailed ? 'warn' : 'error';\n      lc[logLevel]?.(\n        'fetch from API server threw error',\n        {url: finalUrl},\n        error,\n      );\n\n      if (isFetchFailed && (await shouldRetry())) {\n        continue;\n      }\n\n      throw new ProtocolErrorWithLevel(\n        source === 'push'\n          ? {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.ZeroCache,\n              reason: ErrorReason.Internal,\n              message: `Fetch from API server threw error: ${getErrorMessage(error)}`,\n              mutationIDs: [],\n            }\n          : {\n              kind: ErrorKind.TransformFailed,\n              origin: ErrorOrigin.ZeroCache,\n              reason: ErrorReason.Internal,\n              message: `Fetch from API server threw error: ${getErrorMessage(error)}`,\n              queryIDs: [],\n            },\n        logLevel,\n        {cause: error},\n      );\n    }\n  }\n  unreachable();\n}\n\n/**\n * Returns true if the url matches one of the allowedUrlPatterns.\n *\n * URLPattern automatically ignores query parameters and hash fragments during matching\n * because it sets search and hash to wildcard ('*') by default.\n *\n * Example URLPattern patterns:\n * - \"https://api.example.com/endpoint\" - Exact match for a specific URL\n * - \"https://*.example.com/endpoint\" - Matches any single subdomain (e.g., \"https://api.example.com/endpoint\")\n * - \"https://*.*.example.com/endpoint\" - Matches two subdomains (e.g., \"https://api.v1.example.com/endpoint\")\n * - \"https://api.example.com/*\" - Matches any path under /\n * - \"https://api.example.com/:version/endpoint\" - Matches with named parameter (e.g., \"https://api.example.com/v1/endpoint\")\n */\nexport function urlMatch(\n  url: string,\n  allowedUrlPatterns: readonly URLPattern[],\n): boolean {\n  for (const pattern of allowedUrlPatterns) {\n    if (pattern.test(url)) {\n      return true;\n    }\n  }\n  return false;\n}\n\n/**\n * Returns the delay in milliseconds for the next retry attempt using exponential backoff with jitter.\n *\n * The delay assumes the first retry is attempt 1.\n * The formula is: `min(1000, 100 * 2^(attempt - 1) + jitter)` where jitter is between 0 and 100ms.\n */\nfunction getBackoffDelayMs(attempt: number): number {\n  return Math.min(1000, 100 * Math.pow(2, attempt - 1) + Math.random() * 100);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAkBA,IAAM,iBAAiB,CAAC,UAAU,OAAO;;;;;;;;;;AAWzC,SAAgB,kBAAkB,SAA6B;CAC7D,IAAI;EACF,OAAO,IAAI,WAAW,OAAO;CAC/B,SAAS,GAAG;EACV,MAAM,IAAI,MACR,6CAA6C,QAAQ,YAAY,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,GAC5G;CACF;AACF;;;;;;AAOA,SAAS,oBACP,SACA,gBACwB;CACxB,IAAI,CAAC,SACH,OAAO,CAAC;CAEV,IAAI,CAAC,kBAAkB,eAAe,WAAW,GAC/C,OAAO,CAAC;CAGV,MAAM,UAAU,IAAI,IAAI,eAAe,KAAI,MAAK,EAAE,YAAY,CAAC,CAAC;CAChE,MAAM,WAAmC,CAAC;CAC1C,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAC/C,IAAI,QAAQ,IAAI,IAAI,YAAY,CAAC,GAC/B,SAAS,OAAO;CAGpB,OAAO;AACT;AAEA,IAAa,iBAAiB,OAC5B,KACA,OACgC;CAChC,IAAI;EACF,MAAM,OAAO,MAAM,IAAI,MAAM,EAAE,KAAK;EACpC,IAAI,KAAK,SAAS,KAChB,OAAO,KAAK,MAAM,GAAG,GAAG,IAAI;EAE9B,OAAO;CACT,SAAS,GAAG;EACV,GAAG,OACD,8BACA,EACE,KAAK,IAAI,IACX,GACA,CACF;CACF;AAGF;AAEA,IAAM,eAAe;AAErB,eAAsB,mBACpB,WACA,QACA,IACA,KACA,OACA,MACA;CACA,MAAM,uBAAuB,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;CAC5E,KAAK,GACF,YAAY,wBAAwB,oBAAoB,EACxD,YAAY,UAAU,MAAM;CAE/B,MAAM,cAAc,WAAW,SAAS,IAAI,gBAAgB,IAAI;CAChE,MAAM,MAAM,KACV,YAAY,KACZ,oBAAoB,OAAO,gBAC7B;CACA,MAAM,gBAAgB,YAAY;CAElC,GAAG,QAAQ,6BAA6B,EACtC,IACF,CAAC;CAED,IAAI,CAAC,SAAS,KAAK,YAAY,sBAAsB,CAAC,CAAC,GACrD,MAAM,IAAI,uBACR,WAAW,SACP;EACE,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,SAAS,QAAQ,IAAI;EACrB,aAAa,CAAC;CAChB,IACA;EACE,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,SAAS,QAAQ,IAAI;EACrB,UAAU,CAAC;CACb,GACJ,MACF;CAEF,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;CAEA,IAAI,cAAc,QAChB,QAAQ,eAAe,cAAc;CAEvC,OAAO,OACL,SACA,oBACE,cAAc,eACd,cAAc,oBAChB,CACF;CACA,IAAI,IAAI,MAAM,KACZ,QAAQ,mBAAmB,UAAU,IAAI,KAAK;CAEhD,IAAI,cAAc,QAChB,QAAQ,YAAY,cAAc;CAEpC,IAAI,cAAc,QAChB,QAAQ,YAAY,cAAc;CAEpC,YAAY,OAAO,QAAQ,OAAO,GAAG,OAAO;CAE5C,MAAM,SAAS,IAAI,IAAI,GAAG;CAC1B,MAAM,SAAS,IAAI,gBAAgB,OAAO,MAAM;CAEhD,KAAK,MAAM,YAAY,gBACrB,OACE,CAAC,OAAO,IAAI,QAAQ,GACpB,yDAAyD,SAAS,EACpE;CAGF,OAAO,OAAO,UAAU,eAAe,KAAK,CAAC;CAC7C,OAAO,OAAO,SAAS,MAAM,KAAK;CAElC,OAAO,SAAS,OAAO,SAAS;CAEhC,MAAM,WAAW,OAAO,SAAS;CAEjC,KAAK,IAAI,UAAU,GAAG,WAAW,cAAc,WAAW;EACxD,KAAK,GAAG,YAAY,6BAA6B,OAAO;EACxD,GAAG,QAAQ,+BAA+B;EAC1C,MAAM,cAAc,YAAY;GAC9B,IAAI,UAAU,cAAc;IAC1B,MAAM,UAAU,kBAAkB,OAAO;IACzC,GAAG,QAAQ,qCAAqC,QAAQ,IAAI;IAC5D,MAAM,MAAM,OAAO;IACnB,OAAO;GACT;GACA,GAAG,QAAQ,0DAA0D;GACrE,OAAO;EACT;EACA,IAAI;GACF,MAAM,WAAW,MAAM,MAAM,UAAU;IACrC,QAAQ;IACR;IACA,MAAM,KAAK,UAAU,IAAI;GAC3B,CAAC;GAED,IAAI,CAAC,SAAS,IAAI;IAChB,MAAM,cAAc,MAAM,eAAe,UAAU,EAAE;IACrD,GAAG,OAAO,gDAAgD;KACxD,KAAK;KACL,QAAQ,SAAS;KACjB;IACF,CAAC;IAGD,KACG,SAAS,WAAW,OAAO,SAAS,WAAW,QAC/C,MAAM,YAAY,GAEnB;IAGF,MAAM,IAAI,uBACR,WAAW,SACP;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,QAAQ,SAAS;KACjB;KACA,SAAS,gDAAgD,SAAS;KAClE,aAAa,CAAC;IAChB,IACA;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,QAAQ,SAAS;KACjB;KACA,SAAS,gDAAgD,SAAS;KAClE,UAAU,CAAC;IACb,GACJ,MACF;GACF;GAEA,IAAI;IACF,MAAM,OAAO,MAAM,SAAS,KAAK;IACjC,MAAM,SAAS,UAAU,MAAM,MAAM,EACnC,MAAM,cACR,CAAC;IACD,GAAG,QAAQ,iCAAiC;IAC5C,OAAO;GACT,SAAS,OAAO;IACd,GAAG,OACD,4BACA,EACE,KAAK,SACP,GACA,KACF;IAEA,MAAM,IAAI,uBACR,WAAW,SACP;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,SAAS,6CAA6C,gBAAgB,KAAK;KAC3E,aAAa,CAAC;IAChB,IACA;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,SAAS,6CAA6C,gBAAgB,KAAK;KAC3E,UAAU,CAAC;IACb,GACJ,QACA,EAAC,OAAO,MAAK,CACf;GACF;EACF,SAAS,OAAO;GACd,IAAI,gBAAgB,KAAK,GACvB,MAAM;GAGR,MAAM,gBACJ,iBAAiB,aAAa,MAAM,YAAY;GAGlD,IAAI,WAAqB,gBAAgB,SAAS;GAClD,GAAG,YACD,qCACA,EAAC,KAAK,SAAQ,GACd,KACF;GAEA,IAAI,iBAAkB,MAAM,YAAY,GACtC;GAGF,MAAM,IAAI,uBACR,WAAW,SACP;IACE,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,sCAAsC,gBAAgB,KAAK;IACpE,aAAa,CAAC;GAChB,IACA;IACE,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,sCAAsC,gBAAgB,KAAK;IACpE,UAAU,CAAC;GACb,GACJ,UACA,EAAC,OAAO,MAAK,CACf;EACF;CACF;CACA,YAAY;AACd;;;;;;;;;;;;;;AAeA,SAAgB,SACd,KACA,oBACS;CACT,KAAK,MAAM,WAAW,oBACpB,IAAI,QAAQ,KAAK,GAAG,GAClB,OAAO;CAGX,OAAO;AACT;;;;;;;AAQA,SAAS,kBAAkB,SAAyB;CAClD,OAAO,KAAK,IAAI,KAAM,MAAM,KAAK,IAAI,GAAG,UAAU,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG;AAC5E"}
+{"version":3,"file":"fetch.js","names":[],"sources":["../../../../../zero-cache/src/custom/fetch.ts"],"sourcesContent":["import {context, propagation} from '@opentelemetry/api';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport 'urlpattern-polyfill';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../shared/src/error.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport {type Type} from '../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../zero-protocol/src/error-reason.ts';\nimport {isProtocolError} from '../../../zero-protocol/src/error.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {upstreamSchema, type ShardID} from '../types/shards.ts';\n\nconst reservedParams = ['schema', 'appID'];\n\n/**\n * Compiles and validates a URLPattern from configuration.\n *\n * Patterns must be full URLs (e.g., \"https://api.example.com/endpoint\").\n * URLPattern automatically sets search and hash to wildcard ('*'),\n * which means query parameters and fragments are ignored during matching.\n *\n * @throws Error if the pattern is an invalid URLPattern\n */\nexport function compileUrlPattern(pattern: string): URLPattern {\n  try {\n    return new URLPattern(pattern);\n  } catch (e) {\n    throw new Error(\n      `Invalid URLPattern in URL configuration: \"${pattern}\". Error: ${e instanceof Error ? e.message : String(e)}`,\n    );\n  }\n}\n\nexport const getBodyPreview = async (\n  res: Response,\n  lc: LogContext,\n): Promise<string | undefined> => {\n  try {\n    const body = await res.clone().text();\n    if (body.length > 512) {\n      return body.slice(0, 512) + '...';\n    }\n    return body;\n  } catch (e) {\n    lc.warn?.(\n      'failed to get body preview',\n      {\n        url: res.url,\n      },\n      e,\n    );\n  }\n\n  return undefined;\n};\n\nconst MAX_ATTEMPTS = 4;\n\nexport async function fetchFromAPIServer<TValidator extends Type>(\n  validator: TValidator,\n  source: 'push' | 'transform',\n  lc: LogContext,\n  ctx: ConnectionContext,\n  shard: ShardID,\n  body: ReadonlyJSONValue,\n) {\n  const fetchFromAPIServerID = randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n  lc = lc\n    .withContext('fetchFromAPIServerID', fetchFromAPIServerID)\n    .withContext('source', source);\n\n  const fetchConfig = source === 'push' ? ctx.mutateContext : ctx.queryContext;\n  const url = must(\n    fetchConfig.url,\n    `Fetch config for ${source} is missing URL`,\n  );\n  const headerOptions = fetchConfig.headerOptions;\n\n  lc.debug?.('fetchFromAPIServer called', {\n    url,\n  });\n\n  if (!urlMatch(url, fetchConfig.allowedUrlPatterns ?? [])) {\n    throw new ProtocolErrorWithLevel(\n      source === 'push'\n        ? {\n            kind: ErrorKind.PushFailed,\n            origin: ErrorOrigin.ZeroCache,\n            reason: ErrorReason.Internal,\n            message: `URL \"${url}\" is not allowed by the ZERO_MUTATE_URL configuration`,\n            mutationIDs: [],\n          }\n        : {\n            kind: ErrorKind.TransformFailed,\n            origin: ErrorOrigin.ZeroCache,\n            reason: ErrorReason.Internal,\n            message: `URL \"${url}\" is not allowed by the ZERO_QUERY_URL configuration`,\n            queryIDs: [],\n          },\n      'warn',\n    );\n  }\n  const headers: Record<string, string> = {\n    'Content-Type': 'application/json',\n  };\n\n  if (headerOptions.apiKey) {\n    headers['X-Api-Key'] = headerOptions.apiKey;\n  }\n  Object.assign(\n    headers,\n    headerOptions.customHeaders,\n    headerOptions.requestHeaders,\n  );\n  if (ctx.auth?.raw) {\n    headers['Authorization'] = `Bearer ${ctx.auth.raw}`;\n  }\n  if (headerOptions.cookie) {\n    headers['Cookie'] = headerOptions.cookie;\n  }\n  if (headerOptions.origin) {\n    headers['Origin'] = headerOptions.origin;\n  }\n  propagation.inject(context.active(), headers);\n\n  const urlObj = new URL(url);\n  const params = new URLSearchParams(urlObj.search);\n\n  for (const reserved of reservedParams) {\n    assert(\n      !params.has(reserved),\n      `The push URL cannot contain the reserved query param \"${reserved}\"`,\n    );\n  }\n\n  params.append('schema', upstreamSchema(shard));\n  params.append('appID', shard.appID);\n\n  urlObj.search = params.toString();\n\n  const finalUrl = urlObj.toString();\n\n  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {\n    lc = lc.withContext('fetchFromAPIServerAttempt', attempt);\n    lc.debug?.('fetch from API server attempt');\n    const shouldRetry = async () => {\n      if (attempt < MAX_ATTEMPTS) {\n        const delayMs = getBackoffDelayMs(attempt);\n        lc.debug?.(`fetch from API server retrying in ${delayMs} ms`);\n        await sleep(delayMs);\n        return true;\n      }\n      lc.debug?.('fetch from API server reached max attempts, not retrying');\n      return false;\n    };\n    try {\n      const response = await fetch(finalUrl, {\n        method: 'POST',\n        headers,\n        body: JSON.stringify(body),\n      });\n\n      if (!response.ok) {\n        const bodyPreview = await getBodyPreview(response, lc);\n        lc.warn?.('fetch from API server returned non-OK status', {\n          url: finalUrl,\n          status: response.status,\n          bodyPreview,\n        });\n        // Bad Gateway or Gateway Timeout indicate the server was not reached\n        // We retry these if we have retries remaining.\n        if (\n          (response.status === 502 || response.status === 504) &&\n          (await shouldRetry())\n        ) {\n          continue;\n        }\n\n        throw new ProtocolErrorWithLevel(\n          source === 'push'\n            ? {\n                kind: ErrorKind.PushFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.HTTP,\n                status: response.status,\n                bodyPreview,\n                message: `Fetch from API server returned non-OK status ${response.status}`,\n                mutationIDs: [],\n              }\n            : {\n                kind: ErrorKind.TransformFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.HTTP,\n                status: response.status,\n                bodyPreview,\n                message: `Fetch from API server returned non-OK status ${response.status}`,\n                queryIDs: [],\n              },\n          'warn',\n        );\n      }\n\n      try {\n        const json = await response.json();\n        const result = validator.parse(json, {\n          mode: 'passthrough',\n        });\n        lc.debug?.('fetch from API server succeeded');\n        return result;\n      } catch (error) {\n        lc.warn?.(\n          'failed to parse response',\n          {\n            url: finalUrl,\n          },\n          error,\n        );\n\n        throw new ProtocolErrorWithLevel(\n          source === 'push'\n            ? {\n                kind: ErrorKind.PushFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.Parse,\n                message: `Failed to parse response from API server: ${getErrorMessage(error)}`,\n                mutationIDs: [],\n              }\n            : {\n                kind: ErrorKind.TransformFailed,\n                origin: ErrorOrigin.ZeroCache,\n                reason: ErrorReason.Parse,\n                message: `Failed to parse response from API server: ${getErrorMessage(error)}`,\n                queryIDs: [],\n              },\n          'warn',\n          {cause: error},\n        );\n      }\n    } catch (error) {\n      if (isProtocolError(error)) {\n        throw error;\n      }\n\n      const isFetchFailed =\n        error instanceof TypeError && error.message === 'fetch failed';\n      // unexpected/unknown errors should be logged at 'error' level so they\n      // are investigated\n      let logLevel: LogLevel = isFetchFailed ? 'warn' : 'error';\n      lc[logLevel]?.(\n        'fetch from API server threw error',\n        {url: finalUrl},\n        error,\n      );\n\n      if (isFetchFailed && (await shouldRetry())) {\n        continue;\n      }\n\n      throw new ProtocolErrorWithLevel(\n        source === 'push'\n          ? {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.ZeroCache,\n              reason: ErrorReason.Internal,\n              message: `Fetch from API server threw error: ${getErrorMessage(error)}`,\n              mutationIDs: [],\n            }\n          : {\n              kind: ErrorKind.TransformFailed,\n              origin: ErrorOrigin.ZeroCache,\n              reason: ErrorReason.Internal,\n              message: `Fetch from API server threw error: ${getErrorMessage(error)}`,\n              queryIDs: [],\n            },\n        logLevel,\n        {cause: error},\n      );\n    }\n  }\n  unreachable();\n}\n\n/**\n * Returns true if the url matches one of the allowedUrlPatterns.\n *\n * URLPattern automatically ignores query parameters and hash fragments during matching\n * because it sets search and hash to wildcard ('*') by default.\n *\n * Example URLPattern patterns:\n * - \"https://api.example.com/endpoint\" - Exact match for a specific URL\n * - \"https://*.example.com/endpoint\" - Matches any single subdomain (e.g., \"https://api.example.com/endpoint\")\n * - \"https://*.*.example.com/endpoint\" - Matches two subdomains (e.g., \"https://api.v1.example.com/endpoint\")\n * - \"https://api.example.com/*\" - Matches any path under /\n * - \"https://api.example.com/:version/endpoint\" - Matches with named parameter (e.g., \"https://api.example.com/v1/endpoint\")\n */\nexport function urlMatch(\n  url: string,\n  allowedUrlPatterns: readonly URLPattern[],\n): boolean {\n  for (const pattern of allowedUrlPatterns) {\n    if (pattern.test(url)) {\n      return true;\n    }\n  }\n  return false;\n}\n\n/**\n * Returns the delay in milliseconds for the next retry attempt using exponential backoff with jitter.\n *\n * The delay assumes the first retry is attempt 1.\n * The formula is: `min(1000, 100 * 2^(attempt - 1) + jitter)` where jitter is between 0 and 100ms.\n */\nfunction getBackoffDelayMs(attempt: number): number {\n  return Math.min(1000, 100 * Math.pow(2, attempt - 1) + Math.random() * 100);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAkBA,IAAM,iBAAiB,CAAC,UAAU,OAAO;;;;;;;;;;AAWzC,SAAgB,kBAAkB,SAA6B;CAC7D,IAAI;EACF,OAAO,IAAI,WAAW,OAAO;CAC/B,SAAS,GAAG;EACV,MAAM,IAAI,MACR,6CAA6C,QAAQ,YAAY,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,GAC5G;CACF;AACF;AAEA,IAAa,iBAAiB,OAC5B,KACA,OACgC;CAChC,IAAI;EACF,MAAM,OAAO,MAAM,IAAI,MAAM,EAAE,KAAK;EACpC,IAAI,KAAK,SAAS,KAChB,OAAO,KAAK,MAAM,GAAG,GAAG,IAAI;EAE9B,OAAO;CACT,SAAS,GAAG;EACV,GAAG,OACD,8BACA,EACE,KAAK,IAAI,IACX,GACA,CACF;CACF;AAGF;AAEA,IAAM,eAAe;AAErB,eAAsB,mBACpB,WACA,QACA,IACA,KACA,OACA,MACA;CACA,MAAM,uBAAuB,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;CAC5E,KAAK,GACF,YAAY,wBAAwB,oBAAoB,EACxD,YAAY,UAAU,MAAM;CAE/B,MAAM,cAAc,WAAW,SAAS,IAAI,gBAAgB,IAAI;CAChE,MAAM,MAAM,KACV,YAAY,KACZ,oBAAoB,OAAO,gBAC7B;CACA,MAAM,gBAAgB,YAAY;CAElC,GAAG,QAAQ,6BAA6B,EACtC,IACF,CAAC;CAED,IAAI,CAAC,SAAS,KAAK,YAAY,sBAAsB,CAAC,CAAC,GACrD,MAAM,IAAI,uBACR,WAAW,SACP;EACE,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,SAAS,QAAQ,IAAI;EACrB,aAAa,CAAC;CAChB,IACA;EACE,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,SAAS,QAAQ,IAAI;EACrB,UAAU,CAAC;CACb,GACJ,MACF;CAEF,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;CAEA,IAAI,cAAc,QAChB,QAAQ,eAAe,cAAc;CAEvC,OAAO,OACL,SACA,cAAc,eACd,cAAc,cAChB;CACA,IAAI,IAAI,MAAM,KACZ,QAAQ,mBAAmB,UAAU,IAAI,KAAK;CAEhD,IAAI,cAAc,QAChB,QAAQ,YAAY,cAAc;CAEpC,IAAI,cAAc,QAChB,QAAQ,YAAY,cAAc;CAEpC,YAAY,OAAO,QAAQ,OAAO,GAAG,OAAO;CAE5C,MAAM,SAAS,IAAI,IAAI,GAAG;CAC1B,MAAM,SAAS,IAAI,gBAAgB,OAAO,MAAM;CAEhD,KAAK,MAAM,YAAY,gBACrB,OACE,CAAC,OAAO,IAAI,QAAQ,GACpB,yDAAyD,SAAS,EACpE;CAGF,OAAO,OAAO,UAAU,eAAe,KAAK,CAAC;CAC7C,OAAO,OAAO,SAAS,MAAM,KAAK;CAElC,OAAO,SAAS,OAAO,SAAS;CAEhC,MAAM,WAAW,OAAO,SAAS;CAEjC,KAAK,IAAI,UAAU,GAAG,WAAW,cAAc,WAAW;EACxD,KAAK,GAAG,YAAY,6BAA6B,OAAO;EACxD,GAAG,QAAQ,+BAA+B;EAC1C,MAAM,cAAc,YAAY;GAC9B,IAAI,UAAU,cAAc;IAC1B,MAAM,UAAU,kBAAkB,OAAO;IACzC,GAAG,QAAQ,qCAAqC,QAAQ,IAAI;IAC5D,MAAM,MAAM,OAAO;IACnB,OAAO;GACT;GACA,GAAG,QAAQ,0DAA0D;GACrE,OAAO;EACT;EACA,IAAI;GACF,MAAM,WAAW,MAAM,MAAM,UAAU;IACrC,QAAQ;IACR;IACA,MAAM,KAAK,UAAU,IAAI;GAC3B,CAAC;GAED,IAAI,CAAC,SAAS,IAAI;IAChB,MAAM,cAAc,MAAM,eAAe,UAAU,EAAE;IACrD,GAAG,OAAO,gDAAgD;KACxD,KAAK;KACL,QAAQ,SAAS;KACjB;IACF,CAAC;IAGD,KACG,SAAS,WAAW,OAAO,SAAS,WAAW,QAC/C,MAAM,YAAY,GAEnB;IAGF,MAAM,IAAI,uBACR,WAAW,SACP;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,QAAQ,SAAS;KACjB;KACA,SAAS,gDAAgD,SAAS;KAClE,aAAa,CAAC;IAChB,IACA;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,QAAQ,SAAS;KACjB;KACA,SAAS,gDAAgD,SAAS;KAClE,UAAU,CAAC;IACb,GACJ,MACF;GACF;GAEA,IAAI;IACF,MAAM,OAAO,MAAM,SAAS,KAAK;IACjC,MAAM,SAAS,UAAU,MAAM,MAAM,EACnC,MAAM,cACR,CAAC;IACD,GAAG,QAAQ,iCAAiC;IAC5C,OAAO;GACT,SAAS,OAAO;IACd,GAAG,OACD,4BACA,EACE,KAAK,SACP,GACA,KACF;IAEA,MAAM,IAAI,uBACR,WAAW,SACP;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,SAAS,6CAA6C,gBAAgB,KAAK;KAC3E,aAAa,CAAC;IAChB,IACA;KACE,MAAM;KACN,QAAQ;KACR,QAAQ;KACR,SAAS,6CAA6C,gBAAgB,KAAK;KAC3E,UAAU,CAAC;IACb,GACJ,QACA,EAAC,OAAO,MAAK,CACf;GACF;EACF,SAAS,OAAO;GACd,IAAI,gBAAgB,KAAK,GACvB,MAAM;GAGR,MAAM,gBACJ,iBAAiB,aAAa,MAAM,YAAY;GAGlD,IAAI,WAAqB,gBAAgB,SAAS;GAClD,GAAG,YACD,qCACA,EAAC,KAAK,SAAQ,GACd,KACF;GAEA,IAAI,iBAAkB,MAAM,YAAY,GACtC;GAGF,MAAM,IAAI,uBACR,WAAW,SACP;IACE,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,sCAAsC,gBAAgB,KAAK;IACpE,aAAa,CAAC;GAChB,IACA;IACE,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,sCAAsC,gBAAgB,KAAK;IACpE,UAAU,CAAC;GACb,GACJ,UACA,EAAC,OAAO,MAAK,CACf;EACF;CACF;CACA,YAAY;AACd;;;;;;;;;;;;;;AAeA,SAAgB,SACd,KACA,oBACS;CACT,KAAK,MAAM,WAAW,oBACpB,IAAI,QAAQ,KAAK,GAAG,GAClB,OAAO;CAGX,OAAO;AACT;;;;;;;AAQA,SAAS,kBAAkB,SAAyB;CAClD,OAAO,KAAK,IAAI,KAAM,MAAM,KAAK,IAAI,GAAG,UAAU,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG;AAC5E"}

package/out/zero-cache/src/custom-queries/transform-query.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform-query.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/custom-queries/transform-query.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EACL,KAAK,YAAY,EAGlB,MAAM,8CAA8C,CAAC;AAItD,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,4CAA4C,CAAC;AACpD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AAErE,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EAErB,MAAM,uDAAuD,CAAC;AAC/D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,yCAAyC,CAAC;AAC/E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAIhD,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GACD,mBAAmB,CAAC;AAExB,MAAM,MAAM,uBAAuB,GAC/B;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,mBAAmB,CAAC;CAC7B,GACD,CAAC;IACC,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,CAAC,oBAAoB,GAAG,YAAY,CAAC,EAAE,CAAC;CACjD,GAAG,CACA;IACE,MAAM,EAAE,IAAI,CAAC;CACd,GACD;IACE,MAAM,EAAE,KAAK,CAAC;IACd,UAAU,EAAE,oBAAoB,CAAC;CAClC,CACJ,CAAC,CAAC;AAEP;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,sBAAsB;;gBAKrB,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO;IAM1C;;;;;;;;OAQG;IACG,QAAQ,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAM5D,SAAS,CACb,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,QAAQ,CAAC,iBAAiB,CAAC,GACnC,OAAO,CAAC,uBAAuB,CAAC;CAwIpC"}
+{"version":3,"file":"transform-query.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/custom-queries/transform-query.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EACL,KAAK,YAAY,EAGlB,MAAM,8CAA8C,CAAC;AAItD,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,4CAA4C,CAAC;AACpD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AAErE,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACrB,MAAM,uDAAuD,CAAC;AAC/D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,yCAAyC,CAAC;AAC/E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAIhD,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GACD,mBAAmB,CAAC;AAExB,MAAM,MAAM,uBAAuB,GAC/B;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,mBAAmB,CAAC;CAC7B,GACD,CAAC;IACC,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,CAAC,oBAAoB,GAAG,YAAY,CAAC,EAAE,CAAC;CACjD,GAAG,CACA;IACE,MAAM,EAAE,IAAI,CAAC;CACd,GACD;IACE,MAAM,EAAE,KAAK,CAAC;IACd,UAAU,EAAE,oBAAoB,CAAC;CAClC,CACJ,CAAC,CAAC;AAEP;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,sBAAsB;;gBAKrB,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO;IAM1C;;;;;;;;OAQG;IACG,QAAQ,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAM5D,SAAS,CACb,GAAG,EAAE,iBAAiB,EACtB,OAAO,EAAE,QAAQ,CAAC,iBAAiB,CAAC,GACnC,OAAO,CAAC,uBAAuB,CAAC;CAwIpC"}

package/out/zero-cache/src/custom-queries/transform-query.js

@@ -141,14 +141,13 @@ function getCacheKey(ctx, queryID) {
 		origin: ctx.queryContext.headerOptions.origin,
 		userID: ctx.user,
 		url: ctx.queryContext.url,
-		customHeaders: normalizedForwardedHeaders(ctx.queryContext.headerOptions)
+		customHeaders: normalizedHeaders(ctx.queryContext.headerOptions.customHeaders),
+		requestHeaders: normalizedHeaders(ctx.queryContext.headerOptions.requestHeaders)
 	});
 }
-function normalizedForwardedHeaders(headerOptions) {
-	const { allowedClientHeaders, customHeaders } = headerOptions;
-	if (!customHeaders || !allowedClientHeaders || allowedClientHeaders.length === 0) return;
-	const allowedHeaders = new Set(allowedClientHeaders.map((header) => header.toLowerCase()));
-	const forwardedHeaders = sortedEntries(customHeaders).filter(([header]) => allowedHeaders.has(header.toLowerCase()));
+function normalizedHeaders(headers) {
+	if (!headers) return;
+	const forwardedHeaders = sortedEntries(headers);
 	return forwardedHeaders.length === 0 ? void 0 : JSON.stringify(forwardedHeaders);
 }
 //#endregion

package/out/zero-cache/src/custom-queries/transform-query.js.map

@@ -1 +1 @@
-{"version":3,"file":"transform-query.js","names":["#shard","#cache","#lc","#requestTransform"],"sources":["../../../../../zero-cache/src/custom-queries/transform-query.ts"],"sourcesContent":["import {trace} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan} from '../../../otel/src/span.ts';\nimport {TimedCache} from '../../../shared/src/cache.ts';\nimport {getErrorMessage} from '../../../shared/src/error.ts';\nimport {sortedEntries} from '../../../shared/src/sorted-entries.ts';\nimport {\n  type ErroredQuery,\n  type TransformRequestBody,\n  type TransformRequestMessage,\n} from '../../../zero-protocol/src/custom-queries.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type TransformFailedBody,\n} from '../../../zero-protocol/src/error.ts';\nimport {hashOfAST} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n  queryResponseSchema,\n  type QueryResponseBody,\n} from '../../../zero-protocol/src/query-server.ts';\nimport type {TransformedAndHashed} from '../auth/read-authorizer.ts';\nimport {fetchFromAPIServer} from '../custom/fetch.ts';\nimport type {\n  ConnectionContext,\n  ConnectionValidation,\n  HeaderOptions,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport type {ShardID} from '../types/shards.ts';\n\nconst tracer = trace.getTracer('custom-query-transformer');\n\nexport type TransformResponse =\n  | {\n      kind: 'QueryResponse';\n      queries: QueryResponseBody;\n      validation: ConnectionValidation;\n    }\n  | TransformFailedBody;\n\nexport type HashedTransformResponse =\n  | {\n      kind: 'failed';\n      result: TransformFailedBody;\n    }\n  | ({\n      kind: 'success';\n      result: (TransformedAndHashed | ErroredQuery)[];\n    } & (\n      | {\n          cached: true;\n        }\n      | {\n          cached: false;\n          validation: ConnectionValidation;\n        }\n    ));\n\n/**\n * Transforms a custom query by calling the user's API server.\n * Caches the transformed queries for 5 seconds to avoid unnecessary API calls.\n *\n * Error responses are not cached as the user may want to retry the query\n * and the error may be transient.\n *\n * The TTL was chosen to be 5 seconds since custom query requests come with\n * a token which itself may have a short TTL (e.g., 10 seconds).\n *\n * Token expiration isn't expected to be exact so this 5 second\n * caching shouldn't cause unexpected behavior. E.g., many JWT libraries\n * implement leeway for expiration checks: https://github.com/panva/jose/blob/main/docs/jwt/verify/interfaces/JWTVerifyOptions.md#clocktolerance\n *\n * The ViewSyncer will call the API server 3-4 times with the exact same queries\n * if we do not cache requests.\n *\n * Caching is safe here because the cache key encodes the effective request\n * identity used for `/query`: auth, userID, forwarded cookies, origin,\n * custom headers, target URL, and the query hash itself.\n */\nexport class CustomQueryTransformer {\n  readonly #shard: ShardID;\n  readonly #cache: TimedCache<TransformedAndHashed>;\n  readonly #lc: LogContext;\n\n  constructor(lc: LogContext, shard: ShardID) {\n    this.#shard = shard;\n    this.#lc = lc;\n    this.#cache = new TimedCache(5000); // 5 seconds cache TTL\n  }\n\n  /**\n   * Forces the empty `/query` validation request used by auth maintenance.\n   *\n   * This stays separate from `transform()` because `transform([], ...)`\n   * short-circuits locally and never hits the API server, while validation\n   * still needs to make the request so auth failures are surfaced.\n   * Successful validation is intentionally opaque because callers only care\n   * whether the request succeeded or failed.\n   */\n  async validate(ctx: ConnectionContext): Promise<TransformResponse> {\n    const response = await this.#requestTransform(ctx, [], 'validate');\n\n    return response;\n  }\n\n  async transform(\n    ctx: ConnectionContext,\n    queries: Iterable<CustomQueryRecord>,\n  ): Promise<HashedTransformResponse> {\n    const request: TransformRequestBody = [];\n    const cachedResponses: TransformedAndHashed[] = [];\n\n    // split queries into cached and uncached\n    for (const query of queries) {\n      const cacheKey = getCacheKey(ctx, query.id);\n      const cached = this.#cache.get(cacheKey);\n      if (cached) {\n        cachedResponses.push(cached);\n      } else {\n        request.push({\n          id: query.id,\n          name: query.name,\n          args: query.args,\n        });\n      }\n    }\n\n    let cached = true;\n\n    if (request.length === 0) {\n      return {\n        kind: 'success',\n        result: cachedResponses,\n        cached: true,\n      };\n    } else {\n      // we are hitting the server with at least one uncached query\n      cached = false;\n    }\n\n    const response = await this.#requestTransform(ctx, request, 'transform');\n    if (response.kind === 'TransformFailed') {\n      return {\n        kind: 'failed',\n        result: response,\n      };\n    }\n\n    const newResponses = response.queries.map(transformed => {\n      if ('error' in transformed) {\n        return transformed;\n      }\n      return {\n        id: transformed.id,\n        transformedAst: transformed.ast,\n        transformationHash: hashOfAST(transformed.ast),\n      } satisfies TransformedAndHashed;\n    });\n\n    for (const transformed of newResponses) {\n      if ('error' in transformed) {\n        // do not cache error responses\n        continue;\n      }\n      const cacheKey = getCacheKey(ctx, transformed.id);\n      this.#cache.set(cacheKey, transformed);\n    }\n\n    return {\n      kind: 'success',\n      result: [...newResponses, ...cachedResponses],\n      validation: response.validation,\n      cached,\n    };\n  }\n\n  async #requestTransform(\n    ctx: ConnectionContext,\n    request: TransformRequestBody,\n    operation: 'validate' | 'transform',\n  ): Promise<TransformResponse> {\n    const queryIDs = request.map(({id}) => id);\n\n    try {\n      const transformResponse = await startAsyncSpan(\n        tracer,\n        'customQueryTransformer.fetchFromAPIServer',\n        () =>\n          fetchFromAPIServer(\n            queryResponseSchema,\n            'transform',\n            this.#lc,\n            ctx,\n            this.#shard,\n            ['transform', request] satisfies TransformRequestMessage,\n          ),\n      );\n\n      if ('kind' in transformResponse) {\n        return transformResponse.kind === 'QueryResponse'\n          ? {\n              kind: 'QueryResponse',\n              queries: transformResponse.queries,\n              validation:\n                transformResponse.userID !== undefined\n                  ? {\n                      kind: 'server-validated',\n                      validatedUserID: transformResponse.userID,\n                    }\n                  : {kind: 'client-fallback'},\n            }\n          : transformResponse;\n      }\n\n      if (transformResponse[0] === 'transformed') {\n        // Legacy query response, so we use client fallback\n        return {\n          kind: 'QueryResponse',\n          validation: {kind: 'client-fallback'},\n          queries: transformResponse[1],\n        };\n      }\n\n      return transformResponse[1];\n    } catch (e) {\n      if (\n        isProtocolError(e) &&\n        e.errorBody.kind === ErrorKind.TransformFailed\n      ) {\n        return {\n          ...e.errorBody,\n          queryIDs,\n        } as const satisfies TransformFailedBody;\n      }\n\n      return {\n        kind: ErrorKind.TransformFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to ${operation} queries: ${getErrorMessage(e)}`,\n        queryIDs,\n      } as const satisfies TransformFailedBody;\n    }\n  }\n}\n\nfunction getCacheKey(ctx: ConnectionContext, queryID: string) {\n  // For custom queries, queryID is a hash of the name + args.\n  // The apiKey is static for a given transformer instance.\n  return JSON.stringify({\n    queryID,\n    token: ctx.auth?.raw,\n    cookie: ctx.queryContext.headerOptions.cookie,\n    origin: ctx.queryContext.headerOptions.origin,\n    userID: ctx.user,\n    url: ctx.queryContext.url,\n    customHeaders: normalizedForwardedHeaders(ctx.queryContext.headerOptions),\n  });\n}\n\nfunction normalizedForwardedHeaders(headerOptions: HeaderOptions) {\n  const {allowedClientHeaders, customHeaders} = headerOptions;\n  if (\n    !customHeaders ||\n    !allowedClientHeaders ||\n    allowedClientHeaders.length === 0\n  ) {\n    return undefined;\n  }\n\n  const allowedHeaders = new Set(\n    allowedClientHeaders.map(header => header.toLowerCase()),\n  );\n  const forwardedHeaders = sortedEntries(customHeaders).filter(([header]) =>\n    allowedHeaders.has(header.toLowerCase()),\n  );\n\n  return forwardedHeaders.length === 0\n    ? undefined\n    : JSON.stringify(forwardedHeaders);\n}\n"],"mappings":";;;;;;;;;;;;;;AAiCA,IAAM,SAAS,MAAM,UAAU,0BAA0B;;;;;;;;;;;;;;;;;;;;;;AAiDzD,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB;EAC1C,KAAKA,SAAS;EACd,KAAKE,MAAM;EACX,KAAKD,SAAS,IAAI,WAAW,GAAI;CACnC;;;;;;;;;;CAWA,MAAM,SAAS,KAAoD;EAGjE,OAAO,MAFgB,KAAKE,kBAAkB,KAAK,CAAC,GAAG,UAAU;CAGnE;CAEA,MAAM,UACJ,KACA,SACkC;EAClC,MAAM,UAAgC,CAAC;EACvC,MAAM,kBAA0C,CAAC;EAGjD,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,WAAW,YAAY,KAAK,MAAM,EAAE;GAC1C,MAAM,SAAS,KAAKF,OAAO,IAAI,QAAQ;GACvC,IAAI,QACF,gBAAgB,KAAK,MAAM;QAE3B,QAAQ,KAAK;IACX,IAAI,MAAM;IACV,MAAM,MAAM;IACZ,MAAM,MAAM;GACd,CAAC;EAEL;EAEA,IAAI,SAAS;EAEb,IAAI,QAAQ,WAAW,GACrB,OAAO;GACL,MAAM;GACN,QAAQ;GACR,QAAQ;EACV;OAGA,SAAS;EAGX,MAAM,WAAW,MAAM,KAAKE,kBAAkB,KAAK,SAAS,WAAW;EACvE,IAAI,SAAS,SAAS,mBACpB,OAAO;GACL,MAAM;GACN,QAAQ;EACV;EAGF,MAAM,eAAe,SAAS,QAAQ,KAAI,gBAAe;GACvD,IAAI,WAAW,aACb,OAAO;GAET,OAAO;IACL,IAAI,YAAY;IAChB,gBAAgB,YAAY;IAC5B,oBAAoB,UAAU,YAAY,GAAG;GAC/C;EACF,CAAC;EAED,KAAK,MAAM,eAAe,cAAc;GACtC,IAAI,WAAW,aAEb;GAEF,MAAM,WAAW,YAAY,KAAK,YAAY,EAAE;GAChD,KAAKF,OAAO,IAAI,UAAU,WAAW;EACvC;EAEA,OAAO;GACL,MAAM;GACN,QAAQ,CAAC,GAAG,cAAc,GAAG,eAAe;GAC5C,YAAY,SAAS;GACrB;EACF;CACF;CAEA,MAAME,kBACJ,KACA,SACA,WAC4B;EAC5B,MAAM,WAAW,QAAQ,KAAK,EAAC,SAAQ,EAAE;EAEzC,IAAI;GACF,MAAM,oBAAoB,MAAM,eAC9B,QACA,mDAEE,mBACE,qBACA,aACA,KAAKD,KACL,KACA,KAAKF,QACL,CAAC,aAAa,OAAO,CACvB,CACJ;GAEA,IAAI,UAAU,mBACZ,OAAO,kBAAkB,SAAS,kBAC9B;IACE,MAAM;IACN,SAAS,kBAAkB;IAC3B,YACE,kBAAkB,WAAW,KAAA,IACzB;KACE,MAAM;KACN,iBAAiB,kBAAkB;IACrC,IACA,EAAC,MAAM,kBAAiB;GAChC,IACA;GAGN,IAAI,kBAAkB,OAAO,eAE3B,OAAO;IACL,MAAM;IACN,YAAY,EAAC,MAAM,kBAAiB;IACpC,SAAS,kBAAkB;GAC7B;GAGF,OAAO,kBAAkB;EAC3B,SAAS,GAAG;GACV,IACE,gBAAgB,CAAC,KACjB,EAAE,UAAU,SAAS,mBAErB,OAAO;IACL,GAAG,EAAE;IACL;GACF;GAGF,OAAO;IACL,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,aAAa,UAAU,YAAY,gBAAgB,CAAC;IAC7D;GACF;EACF;CACF;AACF;AAEA,SAAS,YAAY,KAAwB,SAAiB;CAG5D,OAAO,KAAK,UAAU;EACpB;EACA,OAAO,IAAI,MAAM;EACjB,QAAQ,IAAI,aAAa,cAAc;EACvC,QAAQ,IAAI,aAAa,cAAc;EACvC,QAAQ,IAAI;EACZ,KAAK,IAAI,aAAa;EACtB,eAAe,2BAA2B,IAAI,aAAa,aAAa;CAC1E,CAAC;AACH;AAEA,SAAS,2BAA2B,eAA8B;CAChE,MAAM,EAAC,sBAAsB,kBAAiB;CAC9C,IACE,CAAC,iBACD,CAAC,wBACD,qBAAqB,WAAW,GAEhC;CAGF,MAAM,iBAAiB,IAAI,IACzB,qBAAqB,KAAI,WAAU,OAAO,YAAY,CAAC,CACzD;CACA,MAAM,mBAAmB,cAAc,aAAa,EAAE,QAAQ,CAAC,YAC7D,eAAe,IAAI,OAAO,YAAY,CAAC,CACzC;CAEA,OAAO,iBAAiB,WAAW,IAC/B,KAAA,IACA,KAAK,UAAU,gBAAgB;AACrC"}
+{"version":3,"file":"transform-query.js","names":["#shard","#cache","#lc","#requestTransform"],"sources":["../../../../../zero-cache/src/custom-queries/transform-query.ts"],"sourcesContent":["import {trace} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {startAsyncSpan} from '../../../otel/src/span.ts';\nimport {TimedCache} from '../../../shared/src/cache.ts';\nimport {getErrorMessage} from '../../../shared/src/error.ts';\nimport {sortedEntries} from '../../../shared/src/sorted-entries.ts';\nimport {\n  type ErroredQuery,\n  type TransformRequestBody,\n  type TransformRequestMessage,\n} from '../../../zero-protocol/src/custom-queries.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type TransformFailedBody,\n} from '../../../zero-protocol/src/error.ts';\nimport {hashOfAST} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n  queryResponseSchema,\n  type QueryResponseBody,\n} from '../../../zero-protocol/src/query-server.ts';\nimport type {TransformedAndHashed} from '../auth/read-authorizer.ts';\nimport {fetchFromAPIServer} from '../custom/fetch.ts';\nimport type {\n  ConnectionContext,\n  ConnectionValidation,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport type {ShardID} from '../types/shards.ts';\n\nconst tracer = trace.getTracer('custom-query-transformer');\n\nexport type TransformResponse =\n  | {\n      kind: 'QueryResponse';\n      queries: QueryResponseBody;\n      validation: ConnectionValidation;\n    }\n  | TransformFailedBody;\n\nexport type HashedTransformResponse =\n  | {\n      kind: 'failed';\n      result: TransformFailedBody;\n    }\n  | ({\n      kind: 'success';\n      result: (TransformedAndHashed | ErroredQuery)[];\n    } & (\n      | {\n          cached: true;\n        }\n      | {\n          cached: false;\n          validation: ConnectionValidation;\n        }\n    ));\n\n/**\n * Transforms a custom query by calling the user's API server.\n * Caches the transformed queries for 5 seconds to avoid unnecessary API calls.\n *\n * Error responses are not cached as the user may want to retry the query\n * and the error may be transient.\n *\n * The TTL was chosen to be 5 seconds since custom query requests come with\n * a token which itself may have a short TTL (e.g., 10 seconds).\n *\n * Token expiration isn't expected to be exact so this 5 second\n * caching shouldn't cause unexpected behavior. E.g., many JWT libraries\n * implement leeway for expiration checks: https://github.com/panva/jose/blob/main/docs/jwt/verify/interfaces/JWTVerifyOptions.md#clocktolerance\n *\n * The ViewSyncer will call the API server 3-4 times with the exact same queries\n * if we do not cache requests.\n *\n * Caching is safe here because the cache key encodes the effective request\n * identity used for `/query`: auth, userID, forwarded cookies, origin,\n * custom headers, target URL, and the query hash itself.\n */\nexport class CustomQueryTransformer {\n  readonly #shard: ShardID;\n  readonly #cache: TimedCache<TransformedAndHashed>;\n  readonly #lc: LogContext;\n\n  constructor(lc: LogContext, shard: ShardID) {\n    this.#shard = shard;\n    this.#lc = lc;\n    this.#cache = new TimedCache(5000); // 5 seconds cache TTL\n  }\n\n  /**\n   * Forces the empty `/query` validation request used by auth maintenance.\n   *\n   * This stays separate from `transform()` because `transform([], ...)`\n   * short-circuits locally and never hits the API server, while validation\n   * still needs to make the request so auth failures are surfaced.\n   * Successful validation is intentionally opaque because callers only care\n   * whether the request succeeded or failed.\n   */\n  async validate(ctx: ConnectionContext): Promise<TransformResponse> {\n    const response = await this.#requestTransform(ctx, [], 'validate');\n\n    return response;\n  }\n\n  async transform(\n    ctx: ConnectionContext,\n    queries: Iterable<CustomQueryRecord>,\n  ): Promise<HashedTransformResponse> {\n    const request: TransformRequestBody = [];\n    const cachedResponses: TransformedAndHashed[] = [];\n\n    // split queries into cached and uncached\n    for (const query of queries) {\n      const cacheKey = getCacheKey(ctx, query.id);\n      const cached = this.#cache.get(cacheKey);\n      if (cached) {\n        cachedResponses.push(cached);\n      } else {\n        request.push({\n          id: query.id,\n          name: query.name,\n          args: query.args,\n        });\n      }\n    }\n\n    let cached = true;\n\n    if (request.length === 0) {\n      return {\n        kind: 'success',\n        result: cachedResponses,\n        cached: true,\n      };\n    } else {\n      // we are hitting the server with at least one uncached query\n      cached = false;\n    }\n\n    const response = await this.#requestTransform(ctx, request, 'transform');\n    if (response.kind === 'TransformFailed') {\n      return {\n        kind: 'failed',\n        result: response,\n      };\n    }\n\n    const newResponses = response.queries.map(transformed => {\n      if ('error' in transformed) {\n        return transformed;\n      }\n      return {\n        id: transformed.id,\n        transformedAst: transformed.ast,\n        transformationHash: hashOfAST(transformed.ast),\n      } satisfies TransformedAndHashed;\n    });\n\n    for (const transformed of newResponses) {\n      if ('error' in transformed) {\n        // do not cache error responses\n        continue;\n      }\n      const cacheKey = getCacheKey(ctx, transformed.id);\n      this.#cache.set(cacheKey, transformed);\n    }\n\n    return {\n      kind: 'success',\n      result: [...newResponses, ...cachedResponses],\n      validation: response.validation,\n      cached,\n    };\n  }\n\n  async #requestTransform(\n    ctx: ConnectionContext,\n    request: TransformRequestBody,\n    operation: 'validate' | 'transform',\n  ): Promise<TransformResponse> {\n    const queryIDs = request.map(({id}) => id);\n\n    try {\n      const transformResponse = await startAsyncSpan(\n        tracer,\n        'customQueryTransformer.fetchFromAPIServer',\n        () =>\n          fetchFromAPIServer(\n            queryResponseSchema,\n            'transform',\n            this.#lc,\n            ctx,\n            this.#shard,\n            ['transform', request] satisfies TransformRequestMessage,\n          ),\n      );\n\n      if ('kind' in transformResponse) {\n        return transformResponse.kind === 'QueryResponse'\n          ? {\n              kind: 'QueryResponse',\n              queries: transformResponse.queries,\n              validation:\n                transformResponse.userID !== undefined\n                  ? {\n                      kind: 'server-validated',\n                      validatedUserID: transformResponse.userID,\n                    }\n                  : {kind: 'client-fallback'},\n            }\n          : transformResponse;\n      }\n\n      if (transformResponse[0] === 'transformed') {\n        // Legacy query response, so we use client fallback\n        return {\n          kind: 'QueryResponse',\n          validation: {kind: 'client-fallback'},\n          queries: transformResponse[1],\n        };\n      }\n\n      return transformResponse[1];\n    } catch (e) {\n      if (\n        isProtocolError(e) &&\n        e.errorBody.kind === ErrorKind.TransformFailed\n      ) {\n        return {\n          ...e.errorBody,\n          queryIDs,\n        } as const satisfies TransformFailedBody;\n      }\n\n      return {\n        kind: ErrorKind.TransformFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to ${operation} queries: ${getErrorMessage(e)}`,\n        queryIDs,\n      } as const satisfies TransformFailedBody;\n    }\n  }\n}\n\nfunction getCacheKey(ctx: ConnectionContext, queryID: string) {\n  // For custom queries, queryID is a hash of the name + args.\n  // The apiKey is static for a given transformer instance.\n  return JSON.stringify({\n    queryID,\n    token: ctx.auth?.raw,\n    cookie: ctx.queryContext.headerOptions.cookie,\n    origin: ctx.queryContext.headerOptions.origin,\n    userID: ctx.user,\n    url: ctx.queryContext.url,\n    customHeaders: normalizedHeaders(\n      ctx.queryContext.headerOptions.customHeaders,\n    ),\n    requestHeaders: normalizedHeaders(\n      ctx.queryContext.headerOptions.requestHeaders,\n    ),\n  });\n}\n\nfunction normalizedHeaders(\n  headers: Readonly<Record<string, string>> | undefined,\n) {\n  if (!headers) {\n    return undefined;\n  }\n\n  const forwardedHeaders = sortedEntries(headers);\n\n  return forwardedHeaders.length === 0\n    ? undefined\n    : JSON.stringify(forwardedHeaders);\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAM,SAAS,MAAM,UAAU,0BAA0B;;;;;;;;;;;;;;;;;;;;;;AAiDzD,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB;EAC1C,KAAKA,SAAS;EACd,KAAKE,MAAM;EACX,KAAKD,SAAS,IAAI,WAAW,GAAI;CACnC;;;;;;;;;;CAWA,MAAM,SAAS,KAAoD;EAGjE,OAAO,MAFgB,KAAKE,kBAAkB,KAAK,CAAC,GAAG,UAAU;CAGnE;CAEA,MAAM,UACJ,KACA,SACkC;EAClC,MAAM,UAAgC,CAAC;EACvC,MAAM,kBAA0C,CAAC;EAGjD,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,WAAW,YAAY,KAAK,MAAM,EAAE;GAC1C,MAAM,SAAS,KAAKF,OAAO,IAAI,QAAQ;GACvC,IAAI,QACF,gBAAgB,KAAK,MAAM;QAE3B,QAAQ,KAAK;IACX,IAAI,MAAM;IACV,MAAM,MAAM;IACZ,MAAM,MAAM;GACd,CAAC;EAEL;EAEA,IAAI,SAAS;EAEb,IAAI,QAAQ,WAAW,GACrB,OAAO;GACL,MAAM;GACN,QAAQ;GACR,QAAQ;EACV;OAGA,SAAS;EAGX,MAAM,WAAW,MAAM,KAAKE,kBAAkB,KAAK,SAAS,WAAW;EACvE,IAAI,SAAS,SAAS,mBACpB,OAAO;GACL,MAAM;GACN,QAAQ;EACV;EAGF,MAAM,eAAe,SAAS,QAAQ,KAAI,gBAAe;GACvD,IAAI,WAAW,aACb,OAAO;GAET,OAAO;IACL,IAAI,YAAY;IAChB,gBAAgB,YAAY;IAC5B,oBAAoB,UAAU,YAAY,GAAG;GAC/C;EACF,CAAC;EAED,KAAK,MAAM,eAAe,cAAc;GACtC,IAAI,WAAW,aAEb;GAEF,MAAM,WAAW,YAAY,KAAK,YAAY,EAAE;GAChD,KAAKF,OAAO,IAAI,UAAU,WAAW;EACvC;EAEA,OAAO;GACL,MAAM;GACN,QAAQ,CAAC,GAAG,cAAc,GAAG,eAAe;GAC5C,YAAY,SAAS;GACrB;EACF;CACF;CAEA,MAAME,kBACJ,KACA,SACA,WAC4B;EAC5B,MAAM,WAAW,QAAQ,KAAK,EAAC,SAAQ,EAAE;EAEzC,IAAI;GACF,MAAM,oBAAoB,MAAM,eAC9B,QACA,mDAEE,mBACE,qBACA,aACA,KAAKD,KACL,KACA,KAAKF,QACL,CAAC,aAAa,OAAO,CACvB,CACJ;GAEA,IAAI,UAAU,mBACZ,OAAO,kBAAkB,SAAS,kBAC9B;IACE,MAAM;IACN,SAAS,kBAAkB;IAC3B,YACE,kBAAkB,WAAW,KAAA,IACzB;KACE,MAAM;KACN,iBAAiB,kBAAkB;IACrC,IACA,EAAC,MAAM,kBAAiB;GAChC,IACA;GAGN,IAAI,kBAAkB,OAAO,eAE3B,OAAO;IACL,MAAM;IACN,YAAY,EAAC,MAAM,kBAAiB;IACpC,SAAS,kBAAkB;GAC7B;GAGF,OAAO,kBAAkB;EAC3B,SAAS,GAAG;GACV,IACE,gBAAgB,CAAC,KACjB,EAAE,UAAU,SAAS,mBAErB,OAAO;IACL,GAAG,EAAE;IACL;GACF;GAGF,OAAO;IACL,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS,aAAa,UAAU,YAAY,gBAAgB,CAAC;IAC7D;GACF;EACF;CACF;AACF;AAEA,SAAS,YAAY,KAAwB,SAAiB;CAG5D,OAAO,KAAK,UAAU;EACpB;EACA,OAAO,IAAI,MAAM;EACjB,QAAQ,IAAI,aAAa,cAAc;EACvC,QAAQ,IAAI,aAAa,cAAc;EACvC,QAAQ,IAAI;EACZ,KAAK,IAAI,aAAa;EACtB,eAAe,kBACb,IAAI,aAAa,cAAc,aACjC;EACA,gBAAgB,kBACd,IAAI,aAAa,cAAc,cACjC;CACF,CAAC;AACH;AAEA,SAAS,kBACP,SACA;CACA,IAAI,CAAC,SACH;CAGF,MAAM,mBAAmB,cAAc,OAAO;CAE9C,OAAO,iBAAiB,WAAW,IAC/B,KAAA,IACA,KAAK,UAAU,gBAAgB;AACrC"}

package/out/zero-cache/src/server/backup-watermark-reader.js

@@ -0,0 +1,35 @@
+import { must } from "../../../shared/src/must.js";
+import { parentWorker, shouldStartWorker, singleProcessMode } from "../types/processes.js";
+import { getZeroConfig } from "../config/zero-config.js";
+import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
+import { createLogContext } from "./logging.js";
+import { VfsBackupWatermarkReader, buildLitestreamVfsReplicaURL } from "../services/litestream/vfs-watermark-reader.js";
+import { VfsBackupWatermarkWorkerService } from "../services/litestream/vfs-watermark-worker.js";
+import { LogContext, consoleLogSink } from "@rocicorp/logger";
+//#region ../zero-cache/src/server/backup-watermark-reader.ts
+var lc = new LogContext("info", {}, consoleLogSink);
+async function runWorker(parent, env, ...argv) {
+	const config = getZeroConfig({
+		env,
+		argv
+	});
+	lc = createLogContext(config, "backup-watermark-reader");
+	const { litestream } = config;
+	const replicaURL = buildLitestreamVfsReplicaURL({
+		backupURL: must(litestream.backupURL, "Missing --litestream-backup-url for backup watermark reader"),
+		endpoint: litestream.endpoint,
+		region: litestream.region
+	});
+	const createReader = () => new VfsBackupWatermarkReader(lc, {
+		replicaURL,
+		extensionPath: litestream.vfsExtensionPath,
+		logLevel: litestream.logLevel,
+		logFile: litestream.vfsLogFile
+	});
+	await runUntilKilled(lc, parent ?? process, new VfsBackupWatermarkWorkerService(lc, parent, createReader, litestream.vfsProbeIntervalMs));
+}
+if (shouldStartWorker(parentWorker, singleProcessMode())) exitAfter(lc, () => runWorker(parentWorker, process.env, ...process.argv.slice(2)));
+//#endregion
+export { runWorker as default };
+
+//# sourceMappingURL=backup-watermark-reader.js.map

package/out/zero-cache/src/server/backup-watermark-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-watermark-reader.js","names":[],"sources":["../../../../../zero-cache/src/server/backup-watermark-reader.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getZeroConfig} from '../config/zero-config.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  buildLitestreamVfsReplicaURL,\n  VfsBackupWatermarkReader,\n} from '../services/litestream/vfs-watermark-reader.ts';\nimport {VfsBackupWatermarkWorkerService} from '../services/litestream/vfs-watermark-worker.ts';\nimport {\n  parentWorker,\n  shouldStartWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\n\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getZeroConfig({env, argv});\n  lc = createLogContext(config, 'backup-watermark-reader');\n  const {litestream} = config;\n\n  const replicaURL = buildLitestreamVfsReplicaURL({\n    backupURL: must(\n      litestream.backupURL,\n      'Missing --litestream-backup-url for backup watermark reader',\n    ),\n    endpoint: litestream.endpoint,\n    region: litestream.region,\n  });\n\n  const createReader = () =>\n    new VfsBackupWatermarkReader(lc, {\n      replicaURL,\n      extensionPath: litestream.vfsExtensionPath,\n      logLevel: litestream.logLevel,\n      logFile: litestream.vfsLogFile,\n    });\n\n  await runUntilKilled(\n    lc,\n    parent ?? process,\n    new VfsBackupWatermarkWorkerService(\n      lc,\n      parent,\n      createReader,\n      litestream.vfsProbeIntervalMs,\n    ),\n  );\n}\n\n// Unlike the other workers, the backup watermark reader is *always* run as its\n// own OS process — forked on demand by the `VfsBackupMonitor` (via\n// `forkChildWorker`), or launched directly as a standalone debug tool — so that\n// the native Litestream VFS extension and its process-global `LITESTREAM_*` env\n// stay isolated from the rest of zero-cache. See `shouldStartWorker`.\nif (shouldStartWorker(parentWorker, singleProcessMode())) {\n  void exitAfter(lc, () =>\n    runWorker(parentWorker, process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;AAiBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,cAAc;EAAC;EAAK;CAAI,CAAC;CACxC,KAAK,iBAAiB,QAAQ,yBAAyB;CACvD,MAAM,EAAC,eAAc;CAErB,MAAM,aAAa,6BAA6B;EAC9C,WAAW,KACT,WAAW,WACX,6DACF;EACA,UAAU,WAAW;EACrB,QAAQ,WAAW;CACrB,CAAC;CAED,MAAM,qBACJ,IAAI,yBAAyB,IAAI;EAC/B;EACA,eAAe,WAAW;EAC1B,UAAU,WAAW;EACrB,SAAS,WAAW;CACtB,CAAC;CAEH,MAAM,eACJ,IACA,UAAU,SACV,IAAI,gCACF,IACA,QACA,cACA,WAAW,kBACb,CACF;AACF;AAOA,IAAI,kBAAkB,cAAc,kBAAkB,CAAC,GACrD,UAAe,UACb,UAAU,cAAc,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC/D"}

package/out/zero-cache/src/server/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA+BA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAQ/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAuMf"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA8BA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAQ/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAkMf"}

package/out/zero-cache/src/server/change-streamer.js

@@ -16,13 +16,13 @@ import { upgradeReplica } from "../services/change-source/common/replica-schema.
 import { ReplicationStatusPublisher, replicationStatusError } from "../services/replicator/replication-status.js";
 import { initializeCustomChangeSource } from "../services/change-source/custom/change-source.js";
 import { initializePostgresChangeSource } from "../services/change-source/pg/change-source.js";
-import { BackupMonitor } from "../services/change-streamer/backup-monitor.js";
 import { ChangeStreamerHttpServer } from "../services/change-streamer/change-streamer-http.js";
+import { BackupNotFoundException, restoreReplica } from "../services/litestream/commands.js";
+import { createBackupCleanupMonitor } from "../services/change-streamer/backup-cleanup-monitor-factory.js";
 import { initChangeStreamerSchema } from "../services/change-streamer/schema/init.js";
 import { PurgeLocker } from "../services/change-streamer/storer.js";
 import { initializeStreamer } from "../services/change-streamer/change-streamer-service.js";
 import { ReplicaMonitor } from "../services/change-streamer/replica-monitor.js";
-import { BackupNotFoundException, getLastBackupTime, restoreReplica } from "../services/litestream/commands.js";
 import { startOtelAuto } from "./otel-start.js";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/change-streamer.ts
@@ -33,7 +33,7 @@ async function runWorker(parent, env, ...argv) {
 		env,
 		argv
 	});
-	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds }, upstream, change, replica, initialSync, litestream, keepaliveTimeoutMs } = config;
+	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds }, upstream, change, replica, initialSync, keepaliveTimeoutMs } = config;
 	startOtelAuto(createLogContext(config, "change-streamer", 0, false), "change-streamer", 0);
 	lc = createLogContext(config, "change-streamer");
 	initEventSink(lc, config);
@@ -62,7 +62,8 @@ async function runWorker(parent, env, ...argv) {
 		changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, purgeLock, autoReset ?? false, {
 			backPressureLimitHeapProportion,
 			flowControlConsensusPaddingSeconds,
-			statementTimeoutMs: change.statementTimeoutMs
+			statementTimeoutMs: change.statementTimeoutMs,
+			changeLogBatchSize: change.logBatchSize
 		}, setTimeout);
 		break;
 	} catch (e) {
@@ -78,13 +79,20 @@ async function runWorker(parent, env, ...argv) {
 	}
 	assert(changeStreamer, `resetting replica did not advance replicaVersion`);
 	await upgradeReplica(lc, "change-streamer-init", replica.file);
-	const { backupURL, port: metricsPort } = litestream;
-	const monitor = backupURL ? new BackupMonitor(lc, replica.file, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, Date.now() - workerStartTime, () => getLastBackupTime(lc, config)) : new ReplicaMonitor(lc, replica.file, changeStreamer);
+	const backupMonitor = createBackupCleanupMonitor({
+		lc,
+		config,
+		replicaFile: replica.file,
+		changeStreamer,
+		initialCleanupDelayMs: Date.now() - workerStartTime,
+		env
+	});
+	const monitor = backupMonitor ?? new ReplicaMonitor(lc, replica.file, changeStreamer);
 	const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
 		port,
 		keepaliveTimeoutMs,
 		startupDelayMs
-	}, parent, changeStreamer, monitor instanceof BackupMonitor ? monitor : null);
+	}, parent, changeStreamer, backupMonitor);
 	parent.send(["ready", { ready: true }]);
 	return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);
 }

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  BackupNotFoundException,\n  getLastBackupTime,\n  restoreReplica,\n} from '../services/litestream/commands.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const workerStartTime = Date.now();\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusPaddingSeconds,\n    },\n    upstream,\n    change,\n    replica,\n    initialSync,\n    litestream,\n    keepaliveTimeoutMs,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  initEventSink(lc, config);\n\n  // Kick off DB connection warmup in the background.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer',\n    {\n      max: change.maxConns,\n    },\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const {autoReset, replicationLag} = config;\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date, then acquire\n  // a lock to prevent change-lock purges. This ensures that (this)\n  // change-streamer will be able to resume from the backup.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();\n\n  // Restore from litestream if the change-log has entries.\n  if (purgeLock) {\n    try {\n      await restoreReplica(lc, config, purgeLock);\n    } catch (e) {\n      // If the restore failed, e.g. due to a corrupt or missing backup, the\n      // replication-manager recovers by re-syncing.\n      const log = e instanceof BackupNotFoundException ? 'warn' : 'error';\n      lc[log]?.(\n        `error restoring backup. resyncing the replica: ${String(e)}`,\n        e,\n      );\n\n      // The purgeLock must be released if the backup could not be restored,\n      // or it will otherwise prevent the change-db update after the resync\n      // completes.\n      await purgeLock.release();\n      purgeLock = null;\n    }\n  }\n\n  let changeStreamer: ChangeStreamerService | undefined;\n\n  const context = getServerContext(config);\n\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {changeSource, subscriptionState} =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        purgeLock,\n        autoReset ?? false,\n        {\n          backPressureLimitHeapProportion,\n          flowControlConsensusPaddingSeconds,\n          statementTimeoutMs: change.statementTimeoutMs,\n        },\n        setTimeout,\n      );\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  // Perform any upgrades to the replica in case it was restored from an\n  // earlier version. Note that this upgrade is done by the replicator worker\n  // as well (in both the replication-manager and the view-syncer), but the\n  // change-streamer independently reads the replica, and it is fine run the\n  // upgrade logic redundantly since it is idempotent.\n  await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n  const {backupURL, port: metricsPort} = litestream;\n  const monitor = backupURL\n    ? new BackupMonitor(\n        lc,\n        replica.file,\n        backupURL,\n        `http://localhost:${metricsPort}/metrics`,\n        changeStreamer,\n        // The time between when the zero-cache was started to when the\n        // change-streamer is ready to start serves as the initial delay for\n        // watermark cleanup (as it either includes a similar replica\n        // restoration/preparation step, or an initial-sync, which\n        // generally takes longer).\n        //\n        // Consider: Also account for permanent volumes?\n        Date.now() - workerStartTime,\n        // Verifies litestream's claimed backup progress against the actual\n        // backup state in the replica destination before advancing the\n        // change-log cleanup watermark.\n        () => getLastBackupTime(lc, config),\n      )\n    : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {port, keepaliveTimeoutMs, startupDelayMs},\n    parent,\n    changeStreamer,\n    monitor instanceof BackupMonitor ? monitor : null,\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer.\n  return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(lc, () =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)).catch(\n      async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      },\n    ),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,kBAAkB,KAAK,IAAI;CACjC,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,YACA,uBACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,cAAc,IAAI,MAAM;CAGxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,mBACA,EACE,KAAK,OAAO,SACd,GACA,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,MAAM;CAKnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,IAAI,YAAY,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ;CAGnE,IAAI,WACF,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,SAAS;CAC5C,SAAS,GAAG;EAGV,MAAM,MAAM,aAAa,0BAA0B,SAAS;EAC5D,GAAG,OACD,kDAAkD,OAAO,CAAC,KAC1D,CACF;EAKA,MAAM,UAAU,QAAQ;EACxB,YAAY;CACd;CAGF,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CAEvC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,gBACjB,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,OACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,WACA,aAAa,OACb;GACE;GACA;GACA,oBAAoB,OAAO;EAC7B,GACA,UACF;EACA;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAYzB,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAOzE,MAAM,eAAe,IAAI,wBAAwB,QAAQ,IAAI;CAE7D,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,QAAQ,MACR,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,IAAI,IAAI,uBAIP,kBAAkB,IAAI,MAAM,CACpC,IACA,IAAI,eAAe,IAAI,QAAQ,MAAM,cAAc;CAEvD,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;CAAc,GACzC,QACA,gBACA,mBAAmB,gBAAgB,UAAU,IAC/C;CAEA,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAIpC,OAAO,eAAe,IAAI,QAAQ,yBAAyB,OAAO;AACpE;AAGA,IAAI,CAAC,kBAAkB,GACrB,UAAe,UACb,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,EAAE,MACnE,OAAM,MAAK;CACT,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CACF,CACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  BackupNotFoundException,\n  restoreReplica,\n} from '../services/litestream/commands.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const workerStartTime = Date.now();\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusPaddingSeconds,\n    },\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  initEventSink(lc, config);\n\n  // Kick off DB connection warmup in the background.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer',\n    {\n      max: change.maxConns,\n    },\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const {autoReset, replicationLag} = config;\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date, then acquire\n  // a lock to prevent change-lock purges. This ensures that (this)\n  // change-streamer will be able to resume from the backup.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();\n\n  // Restore from litestream if the change-log has entries.\n  if (purgeLock) {\n    try {\n      await restoreReplica(lc, config, purgeLock);\n    } catch (e) {\n      // If the restore failed, e.g. due to a corrupt or missing backup, the\n      // replication-manager recovers by re-syncing.\n      const log = e instanceof BackupNotFoundException ? 'warn' : 'error';\n      lc[log]?.(\n        `error restoring backup. resyncing the replica: ${String(e)}`,\n        e,\n      );\n\n      // The purgeLock must be released if the backup could not be restored,\n      // or it will otherwise prevent the change-db update after the resync\n      // completes.\n      await purgeLock.release();\n      purgeLock = null;\n    }\n  }\n\n  let changeStreamer: ChangeStreamerService | undefined;\n\n  const context = getServerContext(config);\n\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {changeSource, subscriptionState} =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        purgeLock,\n        autoReset ?? false,\n        {\n          backPressureLimitHeapProportion,\n          flowControlConsensusPaddingSeconds,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n        },\n        setTimeout,\n      );\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  // Perform any upgrades to the replica in case it was restored from an\n  // earlier version. Note that this upgrade is done by the replicator worker\n  // as well (in both the replication-manager and the view-syncer), but the\n  // change-streamer independently reads the replica, and it is fine run the\n  // upgrade logic redundantly since it is idempotent.\n  await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n    // The time between when the zero-cache was started to when the\n    // change-streamer is ready to start serves as the initial delay for\n    // watermark cleanup (as it either includes a similar replica\n    // restoration/preparation step, or an initial-sync, which\n    // generally takes longer).\n    //\n    // Consider: Also account for permanent volumes?\n    initialCleanupDelayMs: Date.now() - workerStartTime,\n    env,\n  });\n  const monitor =\n    backupMonitor ?? new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {port, keepaliveTimeoutMs, startupDelayMs},\n    parent,\n    changeStreamer,\n    backupMonitor,\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer.\n  return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(lc, () =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)).catch(\n      async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      },\n    ),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwCA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,kBAAkB,KAAK,IAAI;CACjC,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,uBACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,cAAc,IAAI,MAAM;CAGxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,mBACA,EACE,KAAK,OAAO,SACd,GACA,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,MAAM;CAKnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,IAAI,YAAY,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ;CAGnE,IAAI,WACF,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,SAAS;CAC5C,SAAS,GAAG;EAGV,MAAM,MAAM,aAAa,0BAA0B,SAAS;EAC5D,GAAG,OACD,kDAAkD,OAAO,CAAC,KAC1D,CACF;EAKA,MAAM,UAAU,QAAQ;EACxB,YAAY;CACd;CAGF,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CAEvC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,gBACjB,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,OACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,WACA,aAAa,OACb;GACE;GACA;GACA,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;EAC7B,GACA,UACF;EACA;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAYzB,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAOzE,MAAM,eAAe,IAAI,wBAAwB,QAAQ,IAAI;CAE7D,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;EAQA,uBAAuB,KAAK,IAAI,IAAI;EACpC;CACF,CAAC;CACD,MAAM,UACJ,iBAAiB,IAAI,eAAe,IAAI,QAAQ,MAAM,cAAc;CAEtE,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;CAAc,GACzC,QACA,gBACA,aACF;CAEA,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAIpC,OAAO,eAAe,IAAI,QAAQ,yBAAyB,OAAO;AACpE;AAGA,IAAI,CAAC,kBAAkB,GACrB,UAAe,UACb,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,EAAE,MACnE,OAAM,MAAK;CACT,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CACF,CACF"}

package/out/zero-cache/src/server/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAgCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAmC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAuMf"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAgCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAoC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAuMf"}

package/out/zero-cache/src/server/syncer.js

@@ -43,6 +43,7 @@ function getCustomQueryConfig(config) {
 		url: queryConfig.url,
 		apiKey: queryConfig.apiKey,
 		allowedClientHeaders: queryConfig.allowedClientHeaders,
+		allowedRequestHeaders: queryConfig.allowedRequestHeaders,
 		forwardCookies: queryConfig.forwardCookies ?? false
 	};
 }

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('component', 'view-syncer')\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(logger, replicaFile, shard),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(lc, () =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+CA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAGF,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,aAAa,aAAa,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,KAAK,GAC1C,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,MACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,OAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,kBAAkB,GACrB,UAAe,UACb,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CACrE"}
+{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('component', 'view-syncer')\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(logger, replicaFile, shard),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(lc, () =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+CA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAGF,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,aAAa,aAAa,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,KAAK,GAC1C,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,MACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,OAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,kBAAkB,GACrB,UAAe,UACb,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CACrE"}

package/out/zero-cache/src/server/worker-urls.d.ts

@@ -1,4 +1,5 @@
 export declare const CHANGE_STREAMER_URL: URL;
+export declare const BACKUP_WATERMARK_READER_URL: URL;
 export declare const MAIN_URL: URL;
 export declare const MUTATOR_URL: URL;
 export declare const REAPER_URL: URL;

package/out/zero-cache/src/server/worker-urls.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}
+{"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,2BAA2B,KAEvC,CAAC;AACF,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}

package/out/zero-cache/src/server/worker-urls.js

@@ -6,6 +6,7 @@ function resolve(path) {
 	return new URL(path, url);
 }
 var CHANGE_STREAMER_URL = resolve("./change-streamer.ts");
+var BACKUP_WATERMARK_READER_URL = resolve("./backup-watermark-reader.ts");
 var MAIN_URL = resolve("./main.ts");
 resolve("./mutator.ts");
 var REAPER_URL = resolve("./reaper.ts");
@@ -14,6 +15,6 @@ var SHADOW_SYNCER_URL = resolve("./shadow-syncer.ts");
 var SYNCER_URL = resolve("./syncer.ts");
 var WRITE_WORKER_URL = resolve("./write-worker.ts");
 //#endregion
-export { CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SYNCER_URL, WRITE_WORKER_URL };
+export { BACKUP_WATERMARK_READER_URL, CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SYNCER_URL, WRITE_WORKER_URL };
 
 //# sourceMappingURL=worker-urls.js.map

package/out/zero-cache/src/server/worker-urls.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;CACrB,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}
+{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const BACKUP_WATERMARK_READER_URL = resolve(\n  './backup-watermark-reader.ts',\n);\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;CACrB,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,8BAA8B,QACzC,8BACF;AACA,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAO,KAAK,IAAI,EAAE,KAAK,MAAM,EAAC,MAAM,8BAA8B,CAAC;AAK1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAIjD,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM;IACX,OAAO,GAAG;QAAC,GAAG,EAAE,WAAW,CAAC;QAAC,aAAa,EAAE,OAAO,CAAA;KAAC;CACrD,CAAC;AAKF,KAAK,sBAAsB,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG;IAC3C,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAsB,SAAS,CAC7B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,GAAG,EAAE,MAAM,EACX,8BAA8B,SAA6C,EAC3E,eAAe,SAAoB,GAClC,OAAO,CAAC;IACT,QAAQ,EAAE,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAChD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;CACpB,CAAC,CAsGD"}
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAO,KAAK,IAAI,EAAE,KAAK,MAAM,EAAC,MAAM,8BAA8B,CAAC;AAK1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAIjD,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM;IACX,OAAO,GAAG;QAAC,GAAG,EAAE,WAAW,CAAC;QAAC,aAAa,EAAE,OAAO,CAAA;KAAC;CACrD,CAAC;AAKF,KAAK,sBAAsB,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG;IAC3C,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAsB,SAAS,CAC7B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,GAAG,EAAE,MAAM,EACX,8BAA8B,SAA6C,EAC3E,eAAe,SAAoB,GAClC,OAAO,CAAC;IACT,QAAQ,EAAE,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAChD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;CACpB,CAAC,CAkID"}

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -34,11 +34,19 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
 		writable.write(makeAck(lsn));
 		lastAckTime = Date.now();
 	}
+	const inboundTimeoutMs = walSenderTimeoutMs * 2;
+	let lastReceivedTime = Date.now();
 	const livenessTimer = setInterval(() => {
-		if (Date.now() - lastAckTime > manualKeepaliveTimeout) {
+		const now = Date.now();
+		if (now - lastAckTime > manualKeepaliveTimeout) {
 			sendAck(0n);
 			lc.debug?.(`sent manual keepalive`);
 		}
+		const sinceLastReceived = now - lastReceivedTime;
+		if (sinceLastReceived > inboundTimeoutMs && !readable.destroyed) {
+			lc.warn?.(`no message received from ${db.options.host} in ${sinceLastReceived}ms (> 2x wal_sender_timeout ${walSenderTimeoutMs}ms). Destroying replication stream to force reconnect.`);
+			readable.destroy(/* @__PURE__ */ new Error(`replication stream inbound timeout after ${sinceLastReceived}ms`));
+		}
 	}, manualKeepaliveTimeout / 5);
 	let destroyed = false;
 	const parser = new PgoutputParser(await getTypeParsers(db, { returnJsonAsString: true }));
@@ -53,7 +61,10 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
 	pipe({
 		source: readable,
 		sink: messages,
-		parse: (buffer) => parseStreamMessage(lc, buffer, parser),
+		parse: (buffer) => {
+			lastReceivedTime = Date.now();
+			return parseStreamMessage(lc, buffer, parser);
+		},
 		bufferMessages: 5
 	});
 	return {