@rocicorp/zero 1.12.0-canary.1 → 1.12.0-canary.11

package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"inspect-handler.js","names":[],"sources":["../../../../../../zero-cache/src/services/view-syncer/inspect-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {TDigest} from '../../../../shared/src/tdigest.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {AnalyzeQueryResult} from '../../../../zero-protocol/src/analyze-query-result.ts';\nimport type {\n  QueryServerMetrics,\n  ServerMetrics,\n} from '../../../../zero-protocol/src/inspect-down.ts';\nimport {\n  inspectAnalyzeQueryUpSchema,\n  unparsedInspectUpBodySchema,\n  type UnparsedInspectUpMessage,\n} from '../../../../zero-protocol/src/inspect-up.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {loadPermissions} from '../../auth/load-permissions.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {\n  getServerVersion,\n  isAdminPasswordValid,\n} from '../../config/zero-config.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {InspectorDelegate} from '../../server/inspector-delegate.ts';\nimport {analyzeQuery} from '../analyze.ts';\nimport type {ClientHandler} from './client-handler.ts';\nimport type {ConnectionContext} from './connection-context-manager.ts';\nimport type {CVRStore} from './cvr-store.ts';\nimport type {CVRSnapshot} from './cvr.ts';\n\nexport async function handleInspect(\n  lc: LogContext,\n  body: UnparsedInspectUpMessage[1],\n  cvr: CVRSnapshot,\n  client: ClientHandler,\n  inspectorDelegate: InspectorDelegate,\n  clientGroupID: string,\n  // The service handling the request, on whose behalf authentication is\n  // recorded (see InspectorDelegate.clearAuthenticated).\n  authOwner: object,\n  cvrStore: CVRStore,\n  config: NormalizedZeroConfig,\n  ctx: ConnectionContext,\n): Promise<void> {\n  if (!unparsedInspectUpBodySchema.isKnown(body)) {\n    // An op from a newer client. Answer this request only, so the connection\n    // stays up. Nothing is computed for it, so it needs no authentication.\n    const {id} = body;\n    if (typeof id === 'string') {\n      client.sendInspectResponse(lc, {\n        op: 'error',\n        id,\n        value: `Unknown inspector op: ${String(body.op)}`,\n      });\n    }\n    return;\n  }\n\n  // Check if the client is already authenticated. We only authenticate the clientGroup\n  // once per \"worker\".\n  if (\n    body.op !== 'authenticate' &&\n    !inspectorDelegate.isAuthenticated(clientGroupID)\n  ) {\n    lc.info?.(\n      'Client not authenticated to access the inspector protocol. Sending authentication challenge',\n    );\n    client.sendInspectResponse(lc, {\n      op: 'authenticated',\n      id: body.id,\n      value: false,\n    });\n    return;\n  }\n\n  try {\n    switch (body.op) {\n      case 'queries': {\n        const queryRows = await cvrStore.inspectQueries(\n          lc,\n          cvr.ttlClock,\n          body.clientID,\n        );\n\n        // Enhance query rows with server-side materialization metrics\n        const enhancedRows = queryRows.map(row => ({\n          ...row,\n          ast: row.ast ?? inspectorDelegate.getASTForQuery(row.queryID) ?? null,\n          metrics: metricsForProtocol(\n            inspectorDelegate.getMetricsJSONForQuery(row.queryID),\n            ctx.protocolVersion,\n          ),\n        }));\n\n        client.sendInspectResponse(lc, {\n          op: 'queries',\n          id: body.id,\n          value: enhancedRows,\n        });\n        break;\n      }\n\n      case 'metrics': {\n        client.sendInspectResponse(lc, {\n          op: 'metrics',\n          id: body.id,\n          value: inspectorDelegate.getMetricsJSON(),\n        });\n        break;\n      }\n\n      case 'version':\n        client.sendInspectResponse(lc, {\n          op: 'version',\n          id: body.id,\n          value: getServerVersion(config),\n        });\n        break;\n\n      case 'authenticate': {\n        const password = body.value;\n        const ok = isAdminPasswordValid(lc, config, password);\n        if (ok) {\n          inspectorDelegate.setAuthenticated(clientGroupID, authOwner);\n        } else {\n          inspectorDelegate.clearAuthenticated(clientGroupID);\n        }\n\n        client.sendInspectResponse(lc, {\n          op: 'authenticated',\n          id: body.id,\n          value: ok,\n        });\n\n        break;\n      }\n\n      case 'analyze-query': {\n        const parsedBody = v.parse(body, inspectAnalyzeQueryUpSchema, 'strip');\n        let ast = parsedBody.ast ?? parsedBody.value;\n        let legacyQuery = true;\n\n        if (parsedBody.name && parsedBody.args) {\n          // Get the AST from the API server by transforming the named query\n          ast = await inspectorDelegate.transformCustomQuery(\n            parsedBody.name,\n            parsedBody.args,\n            ctx,\n          );\n          legacyQuery = false;\n        }\n\n        if (ast === undefined) {\n          throw new Error(\n            'AST is required for analyze-query operation. Either provide an AST directly or ensure custom query transformation is configured.',\n          );\n        }\n\n        let permissions;\n        if (legacyQuery) {\n          using db = new Database(lc, config.replica.file);\n          const dbRunner = new StatementRunner(db);\n          const loaded = loadPermissions(lc, dbRunner, config.app.id, config);\n          if (loaded.permissions) {\n            permissions = loaded.permissions;\n          } else {\n            lc.info?.(\n              'No permissions loaded; analyze-query will run without applying permissions.',\n            );\n          }\n        }\n\n        const result = await analyzeQuery(\n          lc,\n          config,\n          must(cvr.clientSchema),\n          ast,\n          parsedBody.options?.syncedRows,\n          parsedBody.options?.vendedRows,\n          permissions,\n          ctx.auth?.type === 'jwt' ? ctx.auth : undefined,\n          parsedBody.options?.joinPlans,\n        );\n        client.sendInspectResponse(lc, {\n          op: 'analyze-query',\n          id: body.id,\n          value: analyzeResultForProtocol(result, ctx.protocolVersion),\n        });\n        break;\n      }\n\n      default:\n        unreachable(body);\n    }\n  } catch (e) {\n    lc.warn?.('Error handling inspect message', e);\n    client.sendInspectResponse(lc, {\n      op: 'error',\n      id: body.id,\n      value: (e as Error).message,\n    });\n  }\n}\n\n/**\n * Removes the planner's `filter` nodes from `joinPlans` for clients before\n * protocol 53, which added that node type. 1.9 and 1.10 clients fail to parse\n * a node type they don't know, and the failed parse puts the client in its\n * error state.\n *\n * @visibleForTesting\n */\nexport function analyzeResultForProtocol(\n  result: AnalyzeQueryResult,\n  protocolVersion: number,\n): AnalyzeQueryResult {\n  if (protocolVersion >= 53 || result.joinPlans === undefined) {\n    return result;\n  }\n  return {\n    ...result,\n    joinPlans: result.joinPlans.filter(\n      event => !('nodeType' in event && event.nodeType === 'filter'),\n    ),\n  };\n}\n\n/**\n * Converts per-query server metrics to the appropriate wire format based on the\n * client's protocol version.\n *\n * Protocol >= 51: new format — `query-hydration-server-ms` (plain number) +\n *   `query-update-server` (TDigest).\n * Protocol < 51: old format — `query-materialization-server` (TDigest) +\n *   `query-update-server` (TDigest). The scalar hydration time is wrapped into a\n *   one-point TDigest for backward compatibility.\n *\n * @visibleForTesting\n */\nexport function metricsForProtocol(\n  metrics: QueryServerMetrics | null,\n  protocolVersion: number,\n): QueryServerMetrics | null {\n  if (protocolVersion >= 51 || metrics === null) {\n    return metrics;\n  }\n  // Backward compat: wrap the scalar hydration ms into a one-point TDigest\n  // under the old field name so that 1.5 clients can parse the response.\n  const hydrateDigest = new TDigest();\n  const hydrateMs = metrics['query-hydration-server-ms'];\n  if (hydrateMs !== undefined) {\n    hydrateDigest.add(hydrateMs);\n  }\n  const legacyMetrics: ServerMetrics = {\n    'query-materialization-server': hydrateDigest.toJSON(),\n    'query-update-server': metrics['query-update-server'],\n  };\n  // Cast to QueryServerMetrics: this is intentional — for old-protocol clients\n  // we send the legacy ServerMetrics wire shape under the QueryServerMetrics type\n  // to satisfy TypeScript while preserving backward compatibility.\n  return legacyMetrics as unknown as QueryServerMetrics;\n}\n"],"mappings":";;;;;;;;;;;;AA8BA,eAAsB,cACpB,IACA,MACA,KACA,QACA,mBACA,eAGA,WACA,UACA,QACA,KACe;CACf,IAAI,CAAC,4BAA4B,QAAQ,IAAI,GAAG;EAG9C,MAAM,EAAC,OAAM;EACb,IAAI,OAAO,OAAO,UAChB,OAAO,oBAAoB,IAAI;GAC7B,IAAI;GACJ;GACA,OAAO,yBAAyB,OAAO,KAAK,EAAE;EAChD,CAAC;EAEH;CACF;CAIA,IACE,KAAK,OAAO,kBACZ,CAAC,kBAAkB,gBAAgB,aAAa,GAChD;EACA,GAAG,OACD,6FACF;EACA,OAAO,oBAAoB,IAAI;GAC7B,IAAI;GACJ,IAAI,KAAK;GACT,OAAO;EACT,CAAC;EACD;CACF;CAEA,IAAI;EACF,QAAQ,KAAK,IAAb;GACE,KAAK,WAAW;IAQd,MAAM,gBAAe,MAPG,SAAS,eAC/B,IACA,IAAI,UACJ,KAAK,QACP,GAG+B,KAAI,SAAQ;KACzC,GAAG;KACH,KAAK,IAAI,OAAO,kBAAkB,eAAe,IAAI,OAAO,KAAK;KACjE,SAAS,mBACP,kBAAkB,uBAAuB,IAAI,OAAO,GACpD,IAAI,eACN;IACF,EAAE;IAEF,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO;IACT,CAAC;IACD;GACF;GAEA,KAAK;IACH,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO,kBAAkB,eAAe;IAC1C,CAAC;IACD;GAGF,KAAK;IACH,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO,iBAAiB,MAAM;IAChC,CAAC;IACD;GAEF,KAAK,gBAAgB;IACnB,MAAM,WAAW,KAAK;IACtB,MAAM,KAAK,qBAAqB,IAAI,QAAQ,QAAQ;IACpD,IAAI,IACF,kBAAkB,iBAAiB,eAAe,SAAS;SAE3D,kBAAkB,mBAAmB,aAAa;IAGpD,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO;IACT,CAAC;IAED;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,aAAa,MAAQ,MAAM,6BAA6B,OAAO;IACrE,IAAI,MAAM,WAAW,OAAO,WAAW;IACvC,IAAI,cAAc;IAElB,IAAI,WAAW,QAAQ,WAAW,MAAM;KAEtC,MAAM,MAAM,kBAAkB,qBAC5B,WAAW,MACX,WAAW,MACX,GACF;KACA,cAAc;IAChB;IAEA,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MACR,kIACF;IAGF,IAAI;IACJ,IAAI,aAAa,IAAA;;KAGf,MAAM,SAAS,gBAAgB,IAAI,IADd,gBAAA,YAAA,EADV,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CACV,CACF,GAAU,OAAO,IAAI,IAAI,MAAM;KAClE,IAAI,OAAO,aACT,cAAc,OAAO;UAErB,GAAG,OACD,6EACF;;;;;;IAIJ,MAAM,SAAS,MAAM,aACnB,IACA,QACA,KAAK,IAAI,YAAY,GACrB,KACA,WAAW,SAAS,YACpB,WAAW,SAAS,YACpB,aACA,IAAI,MAAM,SAAS,QAAQ,IAAI,OAAO,KAAA,GACtC,WAAW,SAAS,SACtB;IACA,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO,yBAAyB,QAAQ,IAAI,eAAe;IAC7D,CAAC;IACD;GACF;GAEA,SACE,YAAY,IAAI;EACpB;CACF,SAAS,GAAG;EACV,GAAG,OAAO,kCAAkC,CAAC;EAC7C,OAAO,oBAAoB,IAAI;GAC7B,IAAI;GACJ,IAAI,KAAK;GACT,OAAQ,EAAY;EACtB,CAAC;CACH;AACF;;;;;;;;;AAUA,SAAgB,yBACd,QACA,iBACoB;CACpB,IAAI,mBAAmB,MAAM,OAAO,cAAc,KAAA,GAChD,OAAO;CAET,OAAO;EACL,GAAG;EACH,WAAW,OAAO,UAAU,QAC1B,UAAS,EAAE,cAAc,SAAS,MAAM,aAAa,SACvD;CACF;AACF;;;;;;;;;;;;;AAcA,SAAgB,mBACd,SACA,iBAC2B;CAC3B,IAAI,mBAAmB,MAAM,YAAY,MACvC,OAAO;CAIT,MAAM,gBAAgB,IAAI,QAAQ;CAClC,MAAM,YAAY,QAAQ;CAC1B,IAAI,cAAc,KAAA,GAChB,cAAc,IAAI,SAAS;CAS7B,OAAO;EANL,gCAAgC,cAAc,OAAO;EACrD,uBAAuB,QAAQ;CAK1B;AACT"}
+{"version":3,"file":"inspect-handler.js","names":["v.parse"],"sources":["../../../../../../zero-cache/src/services/view-syncer/inspect-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {TDigest} from '../../../../shared/src/tdigest.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {AnalyzeQueryResult} from '../../../../zero-protocol/src/analyze-query-result.ts';\nimport type {\n  QueryServerMetrics,\n  ServerMetrics,\n} from '../../../../zero-protocol/src/inspect-down.ts';\nimport {\n  inspectAnalyzeQueryUpSchema,\n  unparsedInspectUpBodySchema,\n  type UnparsedInspectUpMessage,\n} from '../../../../zero-protocol/src/inspect-up.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {loadPermissions} from '../../auth/load-permissions.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {\n  getServerVersion,\n  isAdminPasswordValid,\n} from '../../config/zero-config.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {InspectorDelegate} from '../../server/inspector-delegate.ts';\nimport {analyzeQuery} from '../analyze.ts';\nimport type {ClientHandler} from './client-handler.ts';\nimport type {ConnectionContext} from './connection-context-manager.ts';\nimport type {CVRStore} from './cvr-store.ts';\nimport type {CVRSnapshot} from './cvr.ts';\n\nexport async function handleInspect(\n  lc: LogContext,\n  body: UnparsedInspectUpMessage[1],\n  cvr: CVRSnapshot,\n  client: ClientHandler,\n  inspectorDelegate: InspectorDelegate,\n  clientGroupID: string,\n  // The service handling the request, on whose behalf authentication is\n  // recorded (see InspectorDelegate.clearAuthenticated).\n  authOwner: object,\n  cvrStore: CVRStore,\n  config: NormalizedZeroConfig,\n  ctx: ConnectionContext,\n): Promise<void> {\n  if (!unparsedInspectUpBodySchema.isKnown(body)) {\n    // An op from a newer client. Answer this request only, so the connection\n    // stays up. Nothing is computed for it, so it needs no authentication.\n    const {id} = body;\n    if (typeof id === 'string') {\n      client.sendInspectResponse(lc, {\n        op: 'error',\n        id,\n        value: `Unknown inspector op: ${String(body.op)}`,\n      });\n    }\n    return;\n  }\n\n  // Check if the client is already authenticated. We only authenticate the clientGroup\n  // once per \"worker\".\n  if (\n    body.op !== 'authenticate' &&\n    !inspectorDelegate.isAuthenticated(clientGroupID)\n  ) {\n    lc.info?.(\n      'Client not authenticated to access the inspector protocol. Sending authentication challenge',\n    );\n    client.sendInspectResponse(lc, {\n      op: 'authenticated',\n      id: body.id,\n      value: false,\n    });\n    return;\n  }\n\n  try {\n    switch (body.op) {\n      case 'queries': {\n        const queryRows = await cvrStore.inspectQueries(\n          lc,\n          cvr.ttlClock,\n          body.clientID,\n        );\n\n        // Enhance query rows with server-side materialization metrics\n        const enhancedRows = queryRows.map(row => ({\n          ...row,\n          ast: row.ast ?? inspectorDelegate.getASTForQuery(row.queryID) ?? null,\n          metrics: metricsForProtocol(\n            inspectorDelegate.getMetricsJSONForQuery(row.queryID),\n            ctx.protocolVersion,\n          ),\n        }));\n\n        client.sendInspectResponse(lc, {\n          op: 'queries',\n          id: body.id,\n          value: enhancedRows,\n        });\n        break;\n      }\n\n      case 'metrics': {\n        client.sendInspectResponse(lc, {\n          op: 'metrics',\n          id: body.id,\n          value: inspectorDelegate.getMetricsJSON(),\n        });\n        break;\n      }\n\n      case 'version':\n        client.sendInspectResponse(lc, {\n          op: 'version',\n          id: body.id,\n          value: getServerVersion(config),\n        });\n        break;\n\n      case 'authenticate': {\n        const password = body.value;\n        const ok = isAdminPasswordValid(lc, config, password);\n        if (ok) {\n          inspectorDelegate.setAuthenticated(clientGroupID, authOwner);\n        } else {\n          inspectorDelegate.clearAuthenticated(clientGroupID);\n        }\n\n        client.sendInspectResponse(lc, {\n          op: 'authenticated',\n          id: body.id,\n          value: ok,\n        });\n\n        break;\n      }\n\n      case 'analyze-query': {\n        const parsedBody = v.parse(body, inspectAnalyzeQueryUpSchema, 'strip');\n        let ast = parsedBody.ast ?? parsedBody.value;\n        let legacyQuery = true;\n\n        if (parsedBody.name && parsedBody.args) {\n          // Get the AST from the API server by transforming the named query\n          ast = await inspectorDelegate.transformCustomQuery(\n            parsedBody.name,\n            parsedBody.args,\n            ctx,\n          );\n          legacyQuery = false;\n        }\n\n        if (ast === undefined) {\n          throw new Error(\n            'AST is required for analyze-query operation. Either provide an AST directly or ensure custom query transformation is configured.',\n          );\n        }\n\n        let permissions;\n        if (legacyQuery) {\n          using db = new Database(lc, config.replica.file);\n          const dbRunner = new StatementRunner(db);\n          const loaded = loadPermissions(lc, dbRunner, config.app.id, config);\n          if (loaded.permissions) {\n            permissions = loaded.permissions;\n          } else {\n            lc.info?.(\n              'No permissions loaded; analyze-query will run without applying permissions.',\n            );\n          }\n        }\n\n        const result = await analyzeQuery(\n          lc,\n          config,\n          must(cvr.clientSchema),\n          ast,\n          parsedBody.options?.syncedRows,\n          parsedBody.options?.vendedRows,\n          permissions,\n          ctx.auth?.type === 'jwt' ? ctx.auth : undefined,\n          parsedBody.options?.joinPlans,\n        );\n        client.sendInspectResponse(lc, {\n          op: 'analyze-query',\n          id: body.id,\n          value: analyzeResultForProtocol(result, ctx.protocolVersion),\n        });\n        break;\n      }\n\n      default:\n        unreachable(body);\n    }\n  } catch (e) {\n    lc.warn?.('Error handling inspect message', e);\n    client.sendInspectResponse(lc, {\n      op: 'error',\n      id: body.id,\n      value: (e as Error).message,\n    });\n  }\n}\n\n/**\n * Removes the planner's `filter` nodes from `joinPlans` for clients before\n * protocol 53, which added that node type. 1.9 and 1.10 clients fail to parse\n * a node type they don't know, and the failed parse puts the client in its\n * error state.\n *\n * @visibleForTesting\n */\nexport function analyzeResultForProtocol(\n  result: AnalyzeQueryResult,\n  protocolVersion: number,\n): AnalyzeQueryResult {\n  if (protocolVersion >= 53 || result.joinPlans === undefined) {\n    return result;\n  }\n  return {\n    ...result,\n    joinPlans: result.joinPlans.filter(\n      event => !('nodeType' in event && event.nodeType === 'filter'),\n    ),\n  };\n}\n\n/**\n * Converts per-query server metrics to the appropriate wire format based on the\n * client's protocol version.\n *\n * Protocol >= 51: new format — `query-hydration-server-ms` (plain number) +\n *   `query-update-server` (TDigest).\n * Protocol < 51: old format — `query-materialization-server` (TDigest) +\n *   `query-update-server` (TDigest). The scalar hydration time is wrapped into a\n *   one-point TDigest for backward compatibility.\n *\n * @visibleForTesting\n */\nexport function metricsForProtocol(\n  metrics: QueryServerMetrics | null,\n  protocolVersion: number,\n): QueryServerMetrics | null {\n  if (protocolVersion >= 51 || metrics === null) {\n    return metrics;\n  }\n  // Backward compat: wrap the scalar hydration ms into a one-point TDigest\n  // under the old field name so that 1.5 clients can parse the response.\n  const hydrateDigest = new TDigest();\n  const hydrateMs = metrics['query-hydration-server-ms'];\n  if (hydrateMs !== undefined) {\n    hydrateDigest.add(hydrateMs);\n  }\n  const legacyMetrics: ServerMetrics = {\n    'query-materialization-server': hydrateDigest.toJSON(),\n    'query-update-server': metrics['query-update-server'],\n  };\n  // Cast to QueryServerMetrics: this is intentional — for old-protocol clients\n  // we send the legacy ServerMetrics wire shape under the QueryServerMetrics type\n  // to satisfy TypeScript while preserving backward compatibility.\n  return legacyMetrics as unknown as QueryServerMetrics;\n}\n"],"mappings":";;;;;;;;;;;;AA8BA,eAAsB,cACpB,IACA,MACA,KACA,QACA,mBACA,eAGA,WACA,UACA,QACA,KACe;CACf,IAAI,CAAC,4BAA4B,QAAQ,IAAI,GAAG;EAG9C,MAAM,EAAC,OAAM;EACb,IAAI,OAAO,OAAO,UAChB,OAAO,oBAAoB,IAAI;GAC7B,IAAI;GACJ;GACA,OAAO,yBAAyB,OAAO,KAAK,EAAE;EAChD,CAAC;EAEH;CACF;CAIA,IACE,KAAK,OAAO,kBACZ,CAAC,kBAAkB,gBAAgB,aAAa,GAChD;EACA,GAAG,OACD,6FACF;EACA,OAAO,oBAAoB,IAAI;GAC7B,IAAI;GACJ,IAAI,KAAK;GACT,OAAO;EACT,CAAC;EACD;CACF;CAEA,IAAI;EACF,QAAQ,KAAK,IAAb;GACE,KAAK,WAAW;IAQd,MAAM,gBAAe,MAPG,SAAS,eAC/B,IACA,IAAI,UACJ,KAAK,QACP,EAAA,CAG+B,KAAI,SAAQ;KACzC,GAAG;KACH,KAAK,IAAI,OAAO,kBAAkB,eAAe,IAAI,OAAO,KAAK;KACjE,SAAS,mBACP,kBAAkB,uBAAuB,IAAI,OAAO,GACpD,IAAI,eACN;IACF,EAAE;IAEF,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO;IACT,CAAC;IACD;GACF;GAEA,KAAK;IACH,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO,kBAAkB,eAAe;IAC1C,CAAC;IACD;GAGF,KAAK;IACH,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO,iBAAiB,MAAM;IAChC,CAAC;IACD;GAEF,KAAK,gBAAgB;IACnB,MAAM,WAAW,KAAK;IACtB,MAAM,KAAK,qBAAqB,IAAI,QAAQ,QAAQ;IACpD,IAAI,IACF,kBAAkB,iBAAiB,eAAe,SAAS;SAE3D,kBAAkB,mBAAmB,aAAa;IAGpD,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO;IACT,CAAC;IAED;GACF;GAEA,KAAK,iBAAiB;IACpB,MAAM,aAAaA,MAAQ,MAAM,6BAA6B,OAAO;IACrE,IAAI,MAAM,WAAW,OAAO,WAAW;IACvC,IAAI,cAAc;IAElB,IAAI,WAAW,QAAQ,WAAW,MAAM;KAEtC,MAAM,MAAM,kBAAkB,qBAC5B,WAAW,MACX,WAAW,MACX,GACF;KACA,cAAc;IAChB;IAEA,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MACR,kIACF;IAGF,IAAI;IACJ,IAAI,aAAa,IAAA;;KACf,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;KAC/C,MAAM,WAAW,IAAI,gBAAgB,EAAE;KACvC,MAAM,SAAS,gBAAgB,IAAI,UAAU,OAAO,IAAI,IAAI,MAAM;KAClE,IAAI,OAAO,aACT,cAAc,OAAO;UAErB,GAAG,OACD,6EACF;;;;;;IAIJ,MAAM,SAAS,MAAM,aACnB,IACA,QACA,KAAK,IAAI,YAAY,GACrB,KACA,WAAW,SAAS,YACpB,WAAW,SAAS,YACpB,aACA,IAAI,MAAM,SAAS,QAAQ,IAAI,OAAO,KAAA,GACtC,WAAW,SAAS,SACtB;IACA,OAAO,oBAAoB,IAAI;KAC7B,IAAI;KACJ,IAAI,KAAK;KACT,OAAO,yBAAyB,QAAQ,IAAI,eAAe;IAC7D,CAAC;IACD;GACF;GAEA,SACE,YAAY,IAAI;EACpB;CACF,SAAS,GAAG;EACV,GAAG,OAAO,kCAAkC,CAAC;EAC7C,OAAO,oBAAoB,IAAI;GAC7B,IAAI;GACJ,IAAI,KAAK;GACT,OAAQ,EAAY;EACtB,CAAC;CACH;AACF;;;;;;;;;AAUA,SAAgB,yBACd,QACA,iBACoB;CACpB,IAAI,mBAAmB,MAAM,OAAO,cAAc,KAAA,GAChD,OAAO;CAET,OAAO;EACL,GAAG;EACH,WAAW,OAAO,UAAU,QAC1B,UAAS,EAAE,cAAc,SAAS,MAAM,aAAa,SACvD;CACF;AACF;;;;;;;;;;;;;AAcA,SAAgB,mBACd,SACA,iBAC2B;CAC3B,IAAI,mBAAmB,MAAM,YAAY,MACvC,OAAO;CAIT,MAAM,gBAAgB,IAAI,QAAQ;CAClC,MAAM,YAAY,QAAQ;CAC1B,IAAI,cAAc,KAAA,GAChB,cAAc,IAAI,SAAS;CAS7B,OAAO;EANL,gCAAgC,cAAc,OAAO;EACrD,uBAAuB,QAAQ;CAK1B;AACT"}

package/out/zero-cache/src/services/view-syncer/pipeline-driver.js

@@ -2,24 +2,27 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { deepEqual } from "../../../../shared/src/json.js";
 import { getOrInsertComputed } from "../../../../shared/src/map.js";
 import { must } from "../../../../shared/src/must.js";
+import { h32 } from "../../../../shared/src/hash.js";
 import { assignProperty } from "../../../../shared/src/objects.js";
 import { skipYields } from "../../../../zql/src/ivm/skip-yields.js";
 import { throwOutput } from "../../../../zql/src/ivm/operator.js";
 import { MeasurePushOperator } from "../../../../zql/src/query/measure-push-operator.js";
 import { makeSourceChangeAdd, makeSourceChangeEdit, makeSourceChangeRemove } from "../../../../zql/src/ivm/source.js";
-import { buildPipeline } from "../../../../zql/src/builder/builder.js";
+import { buildPipeline, isAggregateTableName, topLevelAggregateTableName } from "../../../../zql/src/builder/builder.js";
+import { stringify } from "../../../../shared/src/bigint-json.js";
 import { Debug, runtimeDebugFlags } from "../../../../zql/src/builder/debug-delegate.js";
 import { TableSource } from "../../../../zqlite/src/table-source.js";
+import { randInt } from "../../../../shared/src/rand.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { computeZqlSpecs, mustGetTableSpec } from "../../db/lite-tables.js";
+import { rowKeyString } from "../../types/row-key.js";
+import { getSubscriptionState } from "../replicator/schema/replication-state.js";
+import { versionToLexi } from "../../types/lexi-version.js";
+import { reloadPermissionsIfChanged } from "../../auth/load-permissions.js";
 import { resolveSimpleScalarSubqueries } from "../../../../zqlite/src/resolve-scalar-subqueries.js";
-import { randInt } from "../../../../shared/src/rand.js";
 import { createSQLiteCostModel } from "../../../../zqlite/src/sqlite-cost-model.js";
-import { reloadPermissionsIfChanged } from "../../auth/load-permissions.js";
 import { LogThrottle } from "../../observability/log-throttle.js";
-import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
-import "../../types/row-key.js";
-import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 import { checkClientSchema } from "./client-schema.js";
 import { planWarningMessage } from "./plan-warnings.js";
 import { PushCutoff } from "./push-cutoff.js";
@@ -42,8 +45,8 @@ var MIN_PROJECTED_ADVANCEMENT_SAMPLE_MS = 5;
 var MIN_PROJECTED_ADVANCEMENT_CHANGES = 16;
 var PROJECTED_ADVANCEMENT_RESET_MULTIPLIER = 1.5;
 var LATE_ADVANCEMENT_FINISH_PROGRESS = .8;
-var planWarningLogThrottle = new LogThrottle({ windowMs: 60 * 6e4 });
-var slowAdvanceLogThrottle = new LogThrottle({ windowMs: 5 * 6e4 });
+var planWarningLogThrottle = new LogThrottle({ windowMs: 36e5 });
+var slowAdvanceLogThrottle = new LogThrottle({ windowMs: 3e5 });
 function randomID() {
 	return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
 }
@@ -82,6 +85,13 @@ function formatAdvancementBreakdown(advanceContext, elapsed) {
 */
 var PipelineDriver = class {
 	#tables = /* @__PURE__ */ new Map();
+	/**
+	* The latest synthetic aggregate row streamed for each key, by synthetic
+	* table. Aggregate tables have no {@link TableSource}, so this is what
+	* {@link getRow} reads for them when the view-syncer rebuilds a put patch on
+	* client catch-up. Entries live as long as the query that produced them.
+	*/
+	#aggregateRows = /* @__PURE__ */ new Map();
 	#pipelines = /* @__PURE__ */ new Map();
 	/**
 	* XOR signature of the set of rows currently attached to each active
@@ -196,6 +206,7 @@ var PipelineDriver = class {
 		this.#abandoned.clear();
 		this.#costBasedAdvanceIncomplete = false;
 		this.#tables.clear();
+		this.#aggregateRows.clear();
 		this.#allTableNames.clear();
 		this.#rowSetSignatures.clear();
 		this.#changedRowSetSignatures.clear();
@@ -205,12 +216,13 @@ var PipelineDriver = class {
 		const { db } = this.#snapshotter.current();
 		const fullTables = /* @__PURE__ */ new Map();
 		computeZqlSpecs(this.#lc, db.db, { includeBackfillingColumns: false }, this.#tableSpecs, fullTables);
-		checkClientSchema(this.#shardID, clientSchema, this.#tableSpecs, fullTables);
+		const tableSpecsWithBackfills = [...fullTables.values()].some((t) => t.backfilling?.length) ? computeZqlSpecs(this.#lc, db.db, { includeBackfillingColumns: true }) : this.#tableSpecs;
+		checkClientSchema(this.#shardID, clientSchema, this.#tableSpecs, fullTables, tableSpecsWithBackfills);
 		this.#allTableNames.clear();
 		for (const table of fullTables.keys()) this.#allTableNames.add(table);
 		const primaryKeys = buildRowKeys(clientSchema, this.#tableSpecs, this.#primaryKeys ?? void 0);
 		this.#primaryKeys = primaryKeys;
-		this.#streamer ??= new Streamer(primaryKeys, this.#tableSpecs);
+		this.#streamer ??= new Streamer(primaryKeys, this.#tableSpecs, () => this.currentVersion(), this.#aggregateRows);
 		const { replicaVersion } = getSubscriptionState(db);
 		this.#replicaVersion = replicaVersion;
 	}
@@ -281,6 +293,7 @@ var PipelineDriver = class {
 			this.#destroyPipeline(queryID, pipeline, "destroy");
 		}
 		this.#tables.clear();
+		this.#aggregateRows.clear();
 		this.#rowSetSignatures.clear();
 		this.#changedRowSetSignatures.clear();
 		this.#storage.destroy();
@@ -671,6 +684,12 @@ var PipelineDriver = class {
 		}
 		this.#rowSetSignatures.delete(queryID);
 		this.#changedRowSetSignatures.delete(queryID);
+		const topLevel = topLevelAggregateTableName(queryID);
+		const prefix = topLevel + ":";
+		for (const [table, byKey] of this.#aggregateRows) if (table === topLevel || table.startsWith(prefix)) {
+			if (stopReason === "remove-query") this.#aggregateRows.delete(table);
+			else for (const entry of byKey.values()) entry.refs = 0;
+		}
 		if (stopReason !== "replace-query") this.#abandoned.delete(queryID);
 		if (stopReason === "remove-query") this.#pushCosts.forget(queryID);
 	}
@@ -747,6 +766,7 @@ var PipelineDriver = class {
 	*/
 	getRow(table, pk) {
 		assert(this.initialized(), "Not yet initialized");
+		if (isAggregateTableName(table)) return this.#aggregateRows.get(table)?.get(rowKeyString(pk))?.row;
 		return must(this.#tables.get(table)).getRow(pk);
 	}
 	/**
@@ -968,8 +988,10 @@ var PipelineDriver = class {
 							if (nextValue) this.#conflictRowsDeleted.add(1);
 							yield* this.#push(tableSource, makeSourceChangeRemove(prevValue));
 						}
-						if (nextValue) if (editOldRow) yield* this.#push(tableSource, makeSourceChangeEdit(nextValue, editOldRow));
-						else yield* this.#push(tableSource, makeSourceChangeAdd(nextValue));
+						if (nextValue) {
+							if (editOldRow) yield* this.#push(tableSource, makeSourceChangeEdit(nextValue, editOldRow));
+							else yield* this.#push(tableSource, makeSourceChangeAdd(nextValue));
+						}
 						pushCompleted = true;
 					} finally {
 						advanceContext.pos++;
@@ -1383,9 +1405,15 @@ var PipelineDriver = class {
 var Streamer = class {
 	#primaryKeys;
 	#tableSpecs;
-	constructor(primaryKeys, tableSpecs) {
+	#version;
+	#aggregateRows;
+	#stampVersion;
+	#stampCount = 0;
+	constructor(primaryKeys, tableSpecs, version, aggregateRows) {
 		this.#primaryKeys = primaryKeys;
 		this.#tableSpecs = tableSpecs;
+		this.#version = version;
+		this.#aggregateRows = aggregateRows;
 	}
 	/** The row changes for a change pushed out of the pipeline of `queryID`. */
 	*streamChange(queryID, schema, change) {
@@ -1414,8 +1442,9 @@ var Streamer = class {
 	*streamNodes(queryID, schema, op, nodes) {
 		const { tableName: table, system } = schema;
 		if (system === "permissions") return;
-		const primaryKey = must(this.#primaryKeys.get(table));
-		const spec = must(this.#tableSpecs.get(table)).tableSpec;
+		const isAggregate = schema.isAggregate ?? false;
+		const primaryKey = isAggregate ? schema.primaryKey : must(this.#primaryKeys.get(table));
+		const minRowVersion = isAggregate ? void 0 : must(this.#tableSpecs.get(table)).tableSpec.minRowVersion;
 		for (const node of nodes) {
 			if (node === "yield") {
 				yield node;
@@ -1425,12 +1454,16 @@ var Streamer = class {
 			let { row } = node;
 			const rowKey = getRowKey(primaryKey, row);
 			if (op !== 1) {
-				const rowVersion = row[ZERO_VERSION_COLUMN_NAME];
-				if (typeof rowVersion === "string" && rowVersion < (spec.minRowVersion ?? "00")) row = {
-					...row,
-					[ZERO_VERSION_COLUMN_NAME]: spec.minRowVersion
-				};
+				if (isAggregate) row = this.#stampAggregateRow(table, rowKey, row, op);
+				else {
+					const rowVersion = row[ZERO_VERSION_COLUMN_NAME];
+					if (typeof rowVersion === "string" && rowVersion < (minRowVersion ?? "00")) row = {
+						...row,
+						[ZERO_VERSION_COLUMN_NAME]: minRowVersion
+					};
+				}
 			}
+			if (op === 1 && isAggregate) this.#releaseAggregateRow(table, rowKeyString(rowKey));
 			yield {
 				type: op,
 				queryID,
@@ -1444,6 +1477,64 @@ var Streamer = class {
 			}
 		}
 	}
+	/**
+	* Stamps a synthetic aggregate row with the replica state version and
+	* remembers it for {@link PipelineDriver.getRow}.
+	*
+	* The CVR only sends a put whose version is strictly greater than the last
+	* one it sent for that row, and the rows of one advance reach it in pages.
+	* An aggregate row is emitted once per changed child, so a stamp is the
+	* replica version plus a count of the stamps made at it, which keeps a row's
+	* stamps strictly increasing however often it is removed and re-added. The
+	* count is a LexiVersion, and a LexiVersion is never a proper prefix of a
+	* later one, so the stamp stays below the next replica version.
+	*
+	* The CVR also takes an equal version to mean equal contents, but the count
+	* starts over in a new process, so the stamp ends in a hash of the contents.
+	*/
+	#stampAggregateRow(table, rowKey, row, op) {
+		const key = rowKeyString(rowKey);
+		const version = this.#version();
+		if (version !== this.#stampVersion) {
+			this.#stampVersion = version;
+			this.#stampCount = 0;
+		}
+		const count = versionToLexi(this.#stampCount++);
+		const hash = h32(stringify(row)).toString(36);
+		const stamped = {
+			...row,
+			[ZERO_VERSION_COLUMN_NAME]: `${version}.${count}.${hash}`
+		};
+		this.#rememberAggregateRow(table, key, stamped, op === 0);
+		return stamped;
+	}
+	#rememberAggregateRow(table, key, row, add) {
+		const rows = this.#aggregateRows;
+		if (!rows) return;
+		let byKey = rows.get(table);
+		if (!byKey) {
+			byKey = /* @__PURE__ */ new Map();
+			rows.set(table, byKey);
+		}
+		const entry = byKey.get(key);
+		if (entry) {
+			entry.row = row;
+			entry.refs += add ? 1 : 0;
+		} else byKey.set(key, {
+			row,
+			refs: add ? 1 : 0
+		});
+	}
+	/**
+	* Drops one reference to an aggregate row. Parents that share a correlation
+	* value share the row, so it is forgotten only when the last one is removed,
+	* as the CVR counts it.
+	*/
+	#releaseAggregateRow(table, key) {
+		const byKey = this.#aggregateRows?.get(table);
+		const entry = byKey?.get(key);
+		if (entry && --entry.refs <= 0) byKey?.delete(key);
+	}
 };
 var QueryFailureLoggingOperator = class {
 	#lc;
@@ -1527,7 +1618,7 @@ function isKeyedBy(rowKey, primaryKey) {
 * of a table outside `clientSchema` are keyed by its key in the replica.
 */
 function hydrate(input, hash, clientSchema, tableSpecs) {
-	return new Streamer(buildRowKeys(clientSchema, tableSpecs), tableSpecs).streamNodes(hash, input.getSchema(), 0, input.fetch({}));
+	return new Streamer(buildRowKeys(clientSchema, tableSpecs), tableSpecs, () => "00").streamNodes(hash, input.getSchema(), 0, input.fetch({}));
 }
 /**
 * Returns the value of `childField` in the row of `input`, the pipeline of a