deepline 0.3.307 → 0.3.309

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts

@@ -11,6 +11,31 @@
  * The API uses only the fixed, tenant-scoped readers in this module.
  */
 import { normalizeRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
+import { createHash } from 'node:crypto';
+import {
+  getRunEventQueryFields,
+  getRunEventQueryPredicate,
+  normalizeRunEventOperationRequest,
+  normalizeRunEventQuery,
+  normalizeRunEventQueryTimestamp,
+  RunEventQueryInputError,
+  RunEventQueryDataError,
+  RUN_EVENT_QUERY_RETENTION_MS,
+  RUN_EVENT_QUERY_MAX_CURSOR_BYTES,
+  RUN_EVENT_QUERY_MAX_REQUEST_BYTES,
+  type NormalizedRunEventQuery,
+  type RunEventOperationRequest,
+  type RunEventOperationResult,
+  type RunEventQueryEvent,
+  type RunEventField,
+  type RunEventPredicate,
+  type RunEventGroupValue,
+  type RunEventScalar,
+  type RunEventQueryPage,
+  type RunEventQueryRetention,
+  type RunEventSummaryResult,
+} from '../run-event-query';
+import { RUN_LOG_LEVELS, type RunLogLevel } from '../run-event-catalog';
 import type { PostgresSchedulerQueryClient } from './postgres';
 import {
   RUN_EVENT_CONVEX_INTERNAL_KIND_SQL_LITERAL,
@@ -464,6 +489,576 @@ export async function searchRuntimeEvents(
   };
 }
 
+type RuntimeRunEventReadEvent = RunEventQueryEvent & {
+  receiptRef: string | null;
+};
+type RunEventQueryRow = EventRow & { cursor_occurred_at: string };
+export type RuntimeRunEventQueryReadResult =
+  | Exclude<RunEventOperationResult, { logs: unknown } | { events: unknown }>
+  | {
+      logs: RuntimeRunEventReadEvent[];
+      page: RunEventQueryPage;
+      query: NormalizedRunEventQuery;
+      retention: RunEventQueryRetention;
+    }
+  | {
+      events: RuntimeRunEventReadEvent[];
+      page: RunEventQueryPage;
+      query: NormalizedRunEventQuery;
+      retention: RunEventQueryRetention;
+    };
+type QueryCursor = {
+  format: 'run-event-query';
+  operation: 'list' | 'summarize';
+  fingerprint: string;
+  query: NormalizedRunEventQuery;
+  position: { occurredAt: string; eventId: string } | RunEventGroupValue[];
+};
+function invalidQueryCursor(): never {
+  throw new RunEventQueryInputError(
+    'Run event cursor does not match this scope, query, resource or projection. Start a new query without cursor.',
+    'invalid_run_event_cursor',
+  );
+}
+function decodeQueryCursor(value: string): QueryCursor {
+  try {
+    if (
+      !/^[A-Za-z0-9_-]+$/.test(value) ||
+      value.length > RUN_EVENT_QUERY_MAX_CURSOR_BYTES
+    )
+      return invalidQueryCursor();
+    const decoded = JSON.parse(
+      Buffer.from(value, 'base64url').toString('utf8'),
+    ) as QueryCursor;
+    if (
+      decoded.format !== 'run-event-query' ||
+      !['list', 'summarize'].includes(decoded.operation) ||
+      typeof decoded.fingerprint !== 'string' ||
+      !/^[a-f0-9]{64}$/.test(decoded.fingerprint) ||
+      !decoded.query ||
+      !decoded.position
+    )
+      return invalidQueryCursor();
+    decoded.query = normalizeRunEventQuery(decoded.query);
+    return decoded;
+  } catch {
+    return invalidQueryCursor();
+  }
+}
+function queryFingerprint(
+  scope: string,
+  org: string,
+  request: RunEventOperationRequest,
+  query: NormalizedRunEventQuery,
+): string {
+  return createHash('sha256')
+    .update(
+      JSON.stringify({
+        scope,
+        org,
+        resource: request.resource,
+        runId: request.runId ?? query.runId ?? null,
+        operation: request.operation,
+        query,
+        order:
+          request.operation === 'summarize'
+            ? 'group_key:asc'
+            : 'occurred_at:event_id:desc',
+        projection:
+          request.operation === 'list' && request.includePayloads === true,
+        groupBy:
+          request.operation === 'summarize' ? request.groupBy : undefined,
+        measures:
+          request.operation === 'summarize' ? request.measures : undefined,
+      }),
+    )
+    .digest('hex');
+}
+function queryFieldSql(field: RunEventField, params: unknown[]): string {
+  const columns: Record<string, string> = {
+    eventId: 'event_id',
+    orgId: 'org_id',
+    runtimeScope: 'runtime_scope',
+    runId: 'run_id',
+    playName: 'play_name',
+    kind: 'kind',
+  };
+  if (columns[field]) return `to_jsonb(${columns[field]})`;
+  if (field === 'occurredAt') return `to_jsonb(${queryTimestampSql()})`;
+  if (field === 'level') return 'to_jsonb(level)';
+  params.push(
+    field.startsWith('context.') ? ['context', field.slice(8)] : [field],
+  );
+  return `search_doc #> $${params.length}::text[]`;
+}
+function queryTimestampSql(): string {
+  return `to_char(occurred_at AT TIME ZONE 'UTC', CASE
+    WHEN mod(extract(microseconds FROM occurred_at)::bigint, 1000) = 0
+      THEN 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'
+      ELSE 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' END)`;
+}
+const QUERY_DIRECT_COLUMNS: Partial<Record<RunEventField, string>> = {
+  eventId: 'event_id',
+  orgId: 'org_id',
+  runtimeScope: 'runtime_scope',
+  runId: 'run_id',
+  playName: 'play_name',
+  kind: 'kind',
+  occurredAt: 'occurred_at',
+  level: 'level',
+};
+function queryLogBodySql(): string {
+  return `CASE WHEN kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
+    AND search_doc->>'source' = 'terminal' THEN
+      regexp_replace(coalesce(log_body, message),
+        '^' || chr(1) || 'prov:[^' || chr(1) || ']*' || chr(1), '')
+    ELSE coalesce(log_body, message) END`;
+}
+function queryPredicateSql(
+  predicate: RunEventPredicate,
+  params: unknown[],
+): string {
+  if ('and' in predicate)
+    return `(${predicate.and.map((term) => queryPredicateSql(term, params)).join(' AND ')})`;
+  if ('or' in predicate)
+    return `(${predicate.or.map((term) => queryPredicateSql(term, params)).join(' OR ')})`;
+  const column = QUERY_DIRECT_COLUMNS[predicate.field];
+  if (column) {
+    if (
+      predicate.field === 'level' &&
+      ['gt', 'gte', 'lt', 'lte'].includes(predicate.op)
+    ) {
+      const index = RUN_LOG_LEVELS.indexOf(predicate.value as RunLogLevel);
+      const levels =
+        predicate.op === 'gt'
+          ? RUN_LOG_LEVELS.slice(index + 1)
+          : predicate.op === 'gte'
+            ? RUN_LOG_LEVELS.slice(index)
+            : predicate.op === 'lt'
+              ? RUN_LOG_LEVELS.slice(0, index)
+              : RUN_LOG_LEVELS.slice(0, index + 1);
+      params.push(levels);
+      return `(${column} = ANY($${params.length}::text[]))`;
+    }
+    if (predicate.op === 'exists')
+      return `(${column} IS ${predicate.value ? 'NOT ' : ''}NULL)`;
+    const bind = (value: RunEventScalar) => {
+      params.push(value);
+      return `$${params.length}${predicate.field === 'occurredAt' ? '::timestamptz' : ''}`;
+    };
+    if (predicate.op === 'in') {
+      const values = predicate.value
+        .filter((value) => value !== null)
+        .map(bind);
+      const terms = values.length
+        ? [`${column} IN (${values.join(', ')})`]
+        : [];
+      if (predicate.value.includes(null)) terms.push(`${column} IS NULL`);
+      return `(${terms.join(' OR ')})`;
+    }
+    if (predicate.value === null)
+      return `(${column} IS ${predicate.op === 'ne' ? 'NOT ' : ''}NULL)`;
+    const operator = {
+      eq: '=',
+      ne: '<>',
+      gt: '>',
+      gte: '>=',
+      lt: '<',
+      lte: '<=',
+    }[predicate.op];
+    return `(${column} ${operator} ${bind(predicate.value)})`;
+  }
+  const field = queryFieldSql(predicate.field, params);
+  if (predicate.op === 'exists')
+    return `(${field} IS ${predicate.value ? 'NOT ' : ''}NULL)`;
+  if (predicate.op === 'in') {
+    const values = predicate.value.map((value) => {
+      params.push(JSON.stringify(value));
+      return `$${params.length}::jsonb`;
+    });
+    return `(${field} IN (${values.join(', ')}))`;
+  }
+  params.push(JSON.stringify(predicate.value));
+  const value = `$${params.length}::jsonb`;
+  const operator = {
+    eq: '=',
+    ne: '<>',
+    gt: '>',
+    gte: '>=',
+    lt: '<',
+    lte: '<=',
+  }[predicate.op];
+  if (predicate.op === 'eq' || predicate.op === 'ne')
+    return `(${field} IS NOT NULL AND ${field} ${operator} ${value})`;
+  return `(jsonb_typeof(${field}) = '${typeof predicate.value === 'number' ? 'number' : 'string'}' AND ${field} ${operator} ${value})`;
+}
+function queryScopeSql(
+  scope: string,
+  org: string,
+  request: RunEventOperationRequest,
+  query: NormalizedRunEventQuery,
+): { params: unknown[]; where: string } {
+  const params: unknown[] = [scope, org, query.since, query.until];
+  // Trusted admission, tenant, runtime and retention predicates always enclose
+  // the complete user expression. An OR can never widen these constraints.
+  const predicates = [
+    'runtime_scope = $1',
+    'org_id = $2',
+    'occurred_at >= $3::timestamptz',
+    'occurred_at < $4::timestamptz',
+    "ingested_at >= clock_timestamp() - interval '7 days'",
+  ];
+  const runId = request.runId ?? query.runId;
+  if (runId !== undefined) {
+    params.push(runId);
+    predicates.push(`run_id = $${params.length}`);
+  }
+  if (request.resource === 'logs') {
+    predicates.push(`kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}`);
+    predicates.push('(log_body IS NOT NULL OR message IS NOT NULL)');
+  }
+  for (const [field, value] of [
+    ['play_name', query.playName],
+    ['kind', query.kind],
+  ] as const) {
+    if (value !== undefined) {
+      params.push(value);
+      predicates.push(`${field} = $${params.length}`);
+    }
+  }
+  if (query.stepId !== undefined) {
+    params.push(query.stepId);
+    predicates.push(`search_doc->'stepId' = to_jsonb($${params.length}::text)`);
+  }
+  if (query.minLevel !== undefined) {
+    params.push(RUN_LOG_LEVELS.slice(RUN_LOG_LEVELS.indexOf(query.minLevel)));
+    predicates.push(`level = ANY($${params.length}::text[])`);
+  }
+  if (query.text !== undefined) {
+    // Escape LIKE metacharacters: this is a literal substring search, never a
+    // user-supplied wildcard, regex or a search over a shortened response tail.
+    params.push(`%${query.text.replace(/[\\%_]/g, '\\$&')}%`);
+    predicates.push(
+      `coalesce((${queryLogBodySql()}), '') ILIKE $${params.length} ESCAPE E'\\\\'`,
+    );
+  }
+  if (query.where)
+    predicates.push(
+      queryPredicateSql(getRunEventQueryPredicate(query.where), params),
+    );
+  return { params, where: predicates.join(' AND ') };
+}
+function exactQueryNumber(value: unknown, label: string): number {
+  const number = Number(value);
+  // SQL sums are exact numeric values. Public fractional results use JS number
+  // precision; integral results must never silently lose integer precision.
+  if (
+    !Number.isFinite(number) ||
+    (Number.isInteger(number) && !Number.isSafeInteger(number)) ||
+    (label === 'count' && (!Number.isSafeInteger(number) || number < 0))
+  )
+    throw new RunEventQueryDataError(
+      `Run event ${label} cannot be represented exactly in the response.`,
+    );
+  return number;
+}
+function flatQueryEvent(row: RunEventQueryRow): RuntimeRunEventReadEvent {
+  const document = row.search_doc;
+  const metadata: Record<string, unknown> = {};
+  for (const field of [
+    'source',
+    'stepId',
+    'status',
+    'stepKind',
+    'datasetId',
+    'phase',
+    'provider',
+    'operation',
+  ] as const) {
+    if (typeof document[field] === 'string') metadata[field] = document[field];
+  }
+  if (typeof document.producerAttempt === 'number')
+    metadata.producerAttempt = document.producerAttempt;
+  if (document.context !== undefined) {
+    if (
+      document.context === null ||
+      typeof document.context !== 'object' ||
+      Array.isArray(document.context) ||
+      Object.values(document.context).some(
+        (value) =>
+          value !== null &&
+          !['string', 'number', 'boolean'].includes(typeof value),
+      )
+    )
+      throw new RunEventQueryDataError(
+        'Run event retained context is not a flat scalar object.',
+      );
+    metadata.context = document.context;
+  }
+  if (row.level !== null && !RUN_LOG_LEVELS.includes(row.level as RunLogLevel))
+    throw new RunEventQueryDataError(
+      'Run event retained level is unsupported.',
+    );
+  if (
+    row.event_payload !== null &&
+    row.event_payload !== undefined &&
+    (typeof row.event_payload !== 'object' || Array.isArray(row.event_payload))
+  )
+    throw new RunEventQueryDataError(
+      'Runtime canonical event payload is not a JSON object.',
+    );
+  return {
+    ...metadata,
+    eventId: row.event_id,
+    occurredAt: normalizeRunEventQueryTimestamp(row.cursor_occurred_at),
+    ingestedAt: iso(row.ingested_at),
+    runId: row.run_id,
+    playName: row.play_name,
+    kind: row.kind,
+    level: row.level as RunLogLevel | null,
+    message: row.message,
+    receiptRef: row.receipt_ref,
+    ...(row.event_payload == null
+      ? {}
+      : { eventPayload: row.event_payload as Record<string, unknown> }),
+  };
+}
+
+/** Reads through the existing restricted reader transaction and 3000ms budget. */
+export async function executeRuntimeEventQuery(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    runtimeScope: string;
+    orgId: string;
+    request: RunEventOperationRequest;
+  },
+): Promise<RuntimeRunEventQueryReadResult> {
+  const scope = requiredScope(input.runtimeScope);
+  const org = requiredOrg(input.orgId);
+  const request = normalizeRunEventOperationRequest(input.request);
+  if (request.operation === 'fields') return getRunEventQueryFields();
+  const cursor =
+    'cursor' in request && request.cursor
+      ? decodeQueryCursor(request.cursor)
+      : undefined;
+  const criteria = Object.fromEntries(
+    Object.entries(request).filter(([key]) =>
+      [
+        'since',
+        'until',
+        'where',
+        'stepId',
+        'minLevel',
+        'text',
+        'playName',
+        'kind',
+        ...(request.resource === 'events' ? ['runId'] : []),
+      ].includes(key),
+    ),
+  );
+  // All omitted criteria are inherited from the cursor's admitted query. Any
+  // supplied criterion must normalize to exactly the original canonical fact.
+  const query = normalizeRunEventQuery({ ...cursor?.query, ...criteria });
+  const fingerprint = queryFingerprint(scope, org, request, query);
+  if (
+    cursor &&
+    (cursor.operation !== request.operation ||
+      cursor.fingerprint !== fingerprint)
+  )
+    invalidQueryCursor();
+  const retention = {
+    retainedSince: new Date(
+      Date.now() - RUN_EVENT_QUERY_RETENTION_MS,
+    ).toISOString(),
+    retentionDays: 7 as const,
+  };
+  const base = { query, retention };
+  const sql = queryScopeSql(scope, org, request, query);
+  if (request.operation === 'count') {
+    const result = await client.query<{ matched_count: string }>(
+      `SELECT count(*)::text AS matched_count FROM ${EVENTS_TABLE} WHERE ${sql.where}`,
+      sql.params,
+    );
+    return {
+      ...base,
+      matchedCount: exactQueryNumber(result.rows[0]?.matched_count, 'count'),
+      exact: true,
+    };
+  }
+  const limit = request.limit!;
+  const encode = (position: QueryCursor['position']) => {
+    const encoded = Buffer.from(
+      JSON.stringify({
+        format: 'run-event-query',
+        operation: request.operation,
+        fingerprint,
+        query,
+        position,
+      }),
+    ).toString('base64url');
+    const continuation = JSON.stringify({
+      ...request,
+      ...query,
+      cursor: encoded,
+    });
+    if (
+      encoded.length > RUN_EVENT_QUERY_MAX_CURSOR_BYTES ||
+      Buffer.byteLength(continuation, 'utf8') >
+        RUN_EVENT_QUERY_MAX_REQUEST_BYTES
+    ) {
+      throw new RunEventQueryDataError(
+        'Run event retained pagination key exceeds the continuation contract.',
+      );
+    }
+    return encoded;
+  };
+  if (request.operation === 'list') {
+    if (cursor) {
+      const position = cursor.position as RuntimeSearchCursor;
+      if (
+        Array.isArray(position) ||
+        typeof position.occurredAt !== 'string' ||
+        typeof position.eventId !== 'string' ||
+        !position.eventId ||
+        position.eventId.length > 512 ||
+        !Number.isFinite(Date.parse(position.occurredAt))
+      )
+        invalidQueryCursor();
+      sql.params.push(position.occurredAt, position.eventId);
+      sql.where += ` AND (occurred_at, event_id) < ($${sql.params.length - 1}::timestamptz, $${sql.params.length})`;
+    }
+    sql.params.push(limit + 1, request.includePayloads === true);
+    const result = await client.query<RunEventQueryRow>(
+      `SELECT event_id, occurred_at, ingested_at, run_id, play_name, kind, level,
+              ${queryTimestampSql()} AS cursor_occurred_at,
+              ${queryLogBodySql()} AS message, receipt_ref, search_doc,
+              CASE WHEN $${sql.params.length}::boolean THEN event_payload END AS event_payload
+         FROM ${EVENTS_TABLE} WHERE ${sql.where}
+         ORDER BY occurred_at DESC, event_id DESC LIMIT $${sql.params.length - 1}::integer`,
+      sql.params,
+    );
+    const hasMore = result.rows.length > limit;
+    const events = result.rows.slice(0, limit).map(flatQueryEvent);
+    const last = result.rows[Math.min(result.rows.length, limit) - 1];
+    const page = {
+      returnedCount: events.length,
+      hasMore,
+      nextCursor:
+        hasMore && last
+          ? encode({
+              occurredAt: last.cursor_occurred_at,
+              eventId: last.event_id,
+            })
+          : null,
+    };
+    return request.resource === 'logs'
+      ? { ...base, logs: events, page }
+      : { ...base, events, page };
+  }
+  const fields = request.groupBy;
+  const measures = request.measures!;
+  const groupFields = fields.map(
+    (field) =>
+      `coalesce(${queryFieldSql(field, sql.params)}, '{"missing":true}'::jsonb)`,
+  );
+  const selected = [
+    `jsonb_build_array(${groupFields.join(', ')}) AS group_key`,
+  ];
+  const invalid: string[] = [];
+  const aggregations = measures.map((measure, i) => {
+    if (measure.op === 'count') return `count(*)::text AS measure_${i}`;
+    const field = queryFieldSql(measure.field, sql.params);
+    selected.push(
+      `CASE WHEN jsonb_typeof(${field}) = 'number' THEN (${field} #>> '{}')::numeric END AS sum_${i}`,
+    );
+    invalid.push(
+      `(${field} IS NOT NULL AND jsonb_typeof(${field}) <> 'number')`,
+    );
+    return `coalesce(sum(sum_${i}), 0)::text AS measure_${i}`;
+  });
+  selected.push(
+    `${invalid.length ? invalid.join(' OR ') : 'false'} AS invalid_sum`,
+  );
+  let continuation = '';
+  if (cursor) {
+    if (
+      !Array.isArray(cursor.position) ||
+      cursor.position.length !== fields.length ||
+      cursor.position.some((value) => {
+        if (
+          value === null ||
+          typeof value === 'string' ||
+          typeof value === 'boolean'
+        )
+          return false;
+        if (typeof value === 'number') return !Number.isFinite(value);
+        return (
+          !value || value.missing !== true || Object.keys(value).length !== 1
+        );
+      })
+    )
+      invalidQueryCursor();
+    sql.params.push(JSON.stringify(cursor.position));
+    continuation = `WHERE group_key > $${sql.params.length}::jsonb`;
+  }
+  sql.params.push(limit + 1);
+  const result = await client.query<Record<string, unknown>>(
+    `WITH matched AS MATERIALIZED (
+       SELECT ${selected.join(', ')} FROM ${EVENTS_TABLE} WHERE ${sql.where}
+     ), grouped AS (
+       SELECT ${fields.length ? 'group_key' : "'[]'::jsonb AS group_key"}, ${aggregations.join(', ')} FROM matched ${fields.length ? 'GROUP BY group_key' : ''}
+     ), validation AS (
+       SELECT count(*) FILTER (WHERE invalid_sum)::text AS invalid_sum_count FROM matched
+     ), paged AS (
+       SELECT * FROM grouped ${continuation} ORDER BY group_key ASC LIMIT $${sql.params.length}::integer
+     ) SELECT paged.*, validation.invalid_sum_count FROM validation LEFT JOIN paged ON true ORDER BY group_key ASC`,
+    sql.params,
+  );
+  if (exactQueryNumber(result.rows[0]?.invalid_sum_count, 'count') > 0) {
+    const sumFields = measures.flatMap((measure) =>
+      measure.op === 'sum' ? [measure.field] : [],
+    );
+    throw new RunEventQueryDataError(
+      `Run event sum selected a nonnumeric value in ${sumFields.join(', ')}. Filter to numeric values or choose another measure.`,
+      'nonnumeric_run_event_sum',
+    );
+  }
+  const rows = result.rows.filter(
+    (row) => row.group_key !== null && row.group_key !== undefined,
+  );
+  const hasMore = rows.length > limit;
+  const groups = rows.slice(0, limit).map((row) => {
+    if (!Array.isArray(row.group_key) || row.group_key.length !== fields.length)
+      throw new RunEventQueryDataError(
+        'Run event group projection is invalid.',
+      );
+    return {
+      keys: Object.fromEntries(
+        fields.map((field, i) => [field, (row.group_key as unknown[])[i]]),
+      ),
+      measures: Object.fromEntries(
+        measures.map((measure, i) => [
+          measure.as!,
+          exactQueryNumber(row[`measure_${i}`], measure.op),
+        ]),
+      ),
+    };
+  });
+  const last = rows[Math.min(rows.length, limit) - 1];
+  return {
+    ...base,
+    groupBy: fields,
+    measures,
+    groups: groups as RunEventSummaryResult['groups'],
+    page: {
+      returnedCount: groups.length,
+      hasMore,
+      nextCursor:
+        hasMore && last ? encode(last.group_key as RunEventGroupValue[]) : null,
+    },
+  };
+}
+
 /**
  * Reads a receipt through a narrow SECURITY DEFINER function. The reader has
  * no direct grant on any scheduler receipt table, and the function checks both