@checkdigit/eslint-athena-plugin 1.0.0-PR.2-dcdf

package/src/athena/api-matcher.ts

@@ -0,0 +1,323 @@
+// athena/api-matcher.ts
+
+import debug from 'debug';
+import type { SchemaObject } from 'ajv/dist/2020';
+
+import type { ApiSchemas, OperationSchemas } from '../openapi/generate-schema';
+import type { Binary, ColumnRefItem, Function as SqlFunction } from './types';
+
+const log = debug('eslint-athena-plugin:athena:api-matcher');
+
+export interface OperationToMatch {
+  path: string;
+  method: string;
+  operationSchemas: OperationSchemas;
+}
+
+export interface MatchedOperation {
+  path: string;
+  method: string;
+  request: SchemaObject;
+  response: SchemaObject;
+}
+
+// A predicate over an API operation candidate: (url path, HTTP method, HTTP response code)
+type OperationPredicate = (
+  path: string,
+  method: string,
+  responseCode: string,
+) => boolean;
+
+const ALWAYS_TRUE: OperationPredicate = () => true;
+
+// --- AST accessor helpers (use Record<string,unknown> to avoid TypeScript narrowing conflicts) ---
+
+function rec(node: unknown): Record<string, unknown> | undefined {
+  return typeof node === 'object' && node !== null
+    ? (node as Record<string, unknown>)
+    : undefined;
+}
+
+function getFunctionName(node: unknown): string | undefined {
+  const fn = rec(node);
+  if (fn?.['type'] !== 'function') {
+    return undefined;
+  }
+  const name = fn['name'] as SqlFunction['name'] | undefined;
+  const firstName = name?.name[0];
+  return firstName === undefined ? undefined : firstName.value.toLowerCase();
+}
+
+function getColumnRef(node: unknown): ColumnRefItem | undefined {
+  const col = rec(node);
+  return col?.['type'] === 'column_ref' ? (node as ColumnRefItem) : undefined;
+}
+
+function getColumnName(node: unknown): string | undefined {
+  const column = getColumnRef(node)?.column;
+  return typeof column === 'string' ? column.toLowerCase() : undefined;
+}
+
+function getColumnTable(node: unknown): string | undefined {
+  return getColumnRef(node)?.table ?? undefined;
+}
+
+function getStringValue(node: unknown): string | undefined {
+  const val = rec(node);
+  if (val?.['type'] !== 'single_quote_string' && val?.['type'] !== 'string') {
+    return undefined;
+  }
+  return typeof val['value'] === 'string' ? val['value'] : undefined;
+}
+
+function getNumberValue(node: unknown): number | undefined {
+  const val = rec(node);
+  return val?.['type'] === 'number' && typeof val['value'] === 'number'
+    ? val['value']
+    : undefined;
+}
+
+// Returns the function args when the node is split(url, '/') or split_part(url, '/'), else undefined.
+function getSplitUrlFunctionArgs(
+  node: unknown,
+  functionName: 'split' | 'split_part',
+): unknown[] | undefined {
+  if (getFunctionName(node) !== functionName) {
+    return undefined;
+  }
+  const args = (rec(node)?.['args'] as { value?: unknown[] } | undefined)
+    ?.value;
+  if (!Array.isArray(args) || args.length < 2) {
+    return undefined;
+  }
+  if (getColumnName(args[0]) !== 'url' || getStringValue(args[1]) !== '/') {
+    return undefined;
+  }
+  return args;
+}
+
+// Matches: split(url, '/')[N]  — a Function node carrying an array_index extension
+interface SplitUrlIndexed extends SqlFunction {
+  array_index: { brackets: true; index: { type: string; value: unknown } }[];
+}
+function isSplitUrlIndexed(node: unknown): node is SplitUrlIndexed {
+  if (getSplitUrlFunctionArgs(node, 'split') === undefined) {
+    return false;
+  }
+  const fn = rec(node);
+  return (
+    Array.isArray(fn?.['array_index']) &&
+    (fn['array_index'] as unknown[]).length > 0
+  );
+}
+
+// Matches: split_part(url, '/', N)  — Presto-style, index in args[2] (1-based)
+function isSplitPartUrl(node: unknown): node is SqlFunction {
+  const args = getSplitUrlFunctionArgs(node, 'split_part');
+  return getNumberValue(args?.[2]) !== undefined;
+}
+
+// Matches: cardinality(split(url, '/'))
+function isCardinalitySplitUrl(node: unknown): node is SqlFunction {
+  if (getFunctionName(node) !== 'cardinality') {
+    return false;
+  }
+  const outerArgs = (rec(node)?.['args'] as { value?: unknown[] } | undefined)
+    ?.value;
+  if (!Array.isArray(outerArgs) || outerArgs.length === 0) {
+    return false;
+  }
+  return getSplitUrlFunctionArgs(outerArgs[0], 'split') !== undefined;
+}
+
+// Returns the table qualifier of the left-hand side of a matchable binary condition.
+// undefined means either unqualified (applies to every table) or indeterminate.
+function getConditionTableQualifier(left: unknown): string | undefined {
+  const colName = getColumnName(left);
+  if (colName !== undefined) {
+    return getColumnTable(left);
+  }
+
+  const splitArgs =
+    getSplitUrlFunctionArgs(left, 'split') ??
+    getSplitUrlFunctionArgs(left, 'split_part');
+  if (splitArgs !== undefined) {
+    return getColumnTable(splitArgs[0]);
+  }
+  if (isCardinalitySplitUrl(left)) {
+    const outerArgs = (rec(left)?.['args'] as { value?: unknown[] } | undefined)
+      ?.value;
+    const splitFn = rec(outerArgs?.[0]);
+    const innerArgs = (splitFn?.['args'] as { value?: unknown[] } | undefined)
+      ?.value;
+    return getColumnTable(innerArgs?.[0]);
+  }
+  return undefined;
+}
+
+// --- Predicate builders ---
+
+// Builds a predicate that checks whether the Nth path segment (1-based) equals a fixed value.
+// Path parameters (`:param`) are treated as wildcards and always match.
+function buildPathSegmentPredicate(
+  index: number,
+  value: string,
+): OperationPredicate {
+  return (path) => {
+    const part = path.split('/')[index - 1];
+    log('checking path segment', { path, index, part, value });
+    return part?.startsWith(':') === true ? true : part === value;
+  };
+}
+
+// tableAlias: the alias (or undefined if none) of the FROM-clause item we are currently matching.
+// Conditions that explicitly reference a different alias are skipped (treated as ALWAYS_TRUE).
+function buildLeafPredicate(
+  node: Binary,
+  tableAlias: string | undefined,
+): OperationPredicate | undefined {
+  if (node.operator !== '=') {
+    return undefined;
+  }
+  const { left, right } = node;
+
+  const conditionTable = getConditionTableQualifier(left);
+  // conditionTable === undefined → unqualified or indeterminate, applies to all tables
+  // conditionTable === string   → qualified; skip if it names a different alias
+  if (
+    conditionTable !== undefined &&
+    tableAlias !== undefined &&
+    conditionTable !== tableAlias
+  ) {
+    return undefined;
+  }
+
+  // method = 'GET'
+  if (getColumnName(left) === 'method') {
+    const value = getStringValue(right);
+    if (value !== undefined) {
+      return (_path, method) => method === value;
+    }
+  }
+
+  // responsestatus = '200'
+  if (getColumnName(left) === 'responsestatus') {
+    const value = getStringValue(right);
+    if (value !== undefined) {
+      return (_path, _method, responseCode) => responseCode === value;
+    }
+  }
+
+  // split(url, '/')[N] = 'value'
+  if (isSplitUrlIndexed(left)) {
+    const index = left.array_index[0]?.index.value;
+    const value = getStringValue(right);
+    if (typeof index === 'number' && value !== undefined) {
+      return buildPathSegmentPredicate(index, value);
+    }
+  }
+
+  // split_part(url, '/', N) = 'value'
+  if (isSplitPartUrl(left)) {
+    const index = getNumberValue(
+      getSplitUrlFunctionArgs(left, 'split_part')?.[2],
+    );
+    const value = getStringValue(right);
+    if (index !== undefined && value !== undefined) {
+      return buildPathSegmentPredicate(index, value);
+    }
+  }
+
+  // cardinality(split(url, '/')) = N
+  if (isCardinalitySplitUrl(left)) {
+    const count = getNumberValue(right);
+    if (count !== undefined) {
+      return (path) => path.split('/').length === count;
+    }
+  }
+
+  return undefined;
+}
+
+function buildPredicate(
+  expr: unknown,
+  tableAlias: string | undefined,
+): OperationPredicate {
+  const node = rec(expr);
+  if (node?.['type'] !== 'binary_expr') {
+    return ALWAYS_TRUE;
+  }
+
+  const binary = expr as Binary;
+
+  switch (binary.operator) {
+    case 'AND': {
+      const leftPred = buildPredicate(binary.left, tableAlias);
+      const rightPred = buildPredicate(binary.right, tableAlias);
+      return (path, method, code) =>
+        leftPred(path, method, code) && rightPred(path, method, code);
+    }
+    case 'OR': {
+      const leftPred = buildPredicate(binary.left, tableAlias);
+      const rightPred = buildPredicate(binary.right, tableAlias);
+      return (path, method, code) =>
+        leftPred(path, method, code) || rightPred(path, method, code);
+    }
+    case 'NOT': {
+      const innerPred = buildPredicate(binary.left, tableAlias);
+      return (path, method, code) => !innerPred(path, method, code);
+    }
+    default: {
+      return buildLeafPredicate(binary, tableAlias) ?? ALWAYS_TRUE;
+    }
+  }
+}
+
+export function matchApi(
+  selectAST: object,
+  tableAST: object,
+  apiSchemas: ApiSchemas[],
+): MatchedOperation[] | undefined {
+  const tableAlias = (tableAST as { as?: string | null }).as ?? undefined;
+  const predicate = buildPredicate(
+    (selectAST as { where?: unknown }).where,
+    tableAlias,
+  );
+
+  const allOperations: OperationToMatch[] = apiSchemas
+    .flatMap((apiSchema) => Object.entries(apiSchema.apis))
+    .flatMap(([path, operations]) =>
+      Object.entries(operations).map(([method, operationSchemas]) => ({
+        path,
+        method: method.toUpperCase(),
+        operationSchemas,
+      })),
+    );
+  log('total operation schemas', allOperations.length);
+
+  const matchedApis = allOperations.flatMap(
+    ({ path, method, operationSchemas }) =>
+      Object.entries(operationSchemas.responses).flatMap(
+        ([responseCode, responseSchema]) =>
+          predicate(path, method, responseCode)
+            ? [
+                {
+                  path,
+                  method,
+                  request: operationSchemas.request,
+                  response: responseSchema,
+                },
+              ]
+            : [],
+      ),
+  );
+  log('matched apis', matchedApis.length);
+
+  if (matchedApis.length === 0) {
+    log('no matched api');
+    throw new Error(
+      'No matched api, please adjust your query conditions to match with at least one API endpoints with firehose enabled.',
+    );
+  }
+  return matchedApis;
+}

package/src/athena/athena.ts

@@ -0,0 +1,224 @@
+// athena/athena.ts
+
+/*
+ * Copyright (c) 2021-2026 Check Digit, LLC
+ *
+ * This code is licensed under the MIT license (see LICENSE.txt for details).
+ */
+
+import debug from 'debug';
+import {
+  AST_NODE_TYPES,
+  ESLintUtils,
+  type TSESTree,
+} from '@typescript-eslint/utils';
+
+import { parse } from '../peggy/athena-peggy.ts';
+import type { AST } from './types';
+import { createRootContext } from './context.ts';
+import {
+  ATHENA_ERROR,
+  AthenaError,
+  checkAthenaAst,
+  offsetToLoc,
+  SYNTAXT_ERROR,
+} from './validate.ts';
+
+export const ruleId = 'athena';
+
+const log = debug('eslint-athena-plugin:athena');
+const createRule = ESLintUtils.RuleCreator((name) => name);
+
+// Maps a SQL-string offset (as produced by the PEG parser) back to an absolute source offset.
+// Each quasi in a TemplateLiteral contributes a segment; template expressions have no SQL width
+// but do occupy source characters, so a naïve single-offset approach gives wrong results when
+// the error location is in a quasi that follows one or more template expressions.
+interface SqlSourceSegment {
+  sqlStart: number; // offset in the trimmed SQL where this quasi's content begins
+  srcStart: number; // absolute source offset where this quasi's content begins
+}
+
+function buildSqlMapping(sqlNode: TSESTree.Node): SqlSourceSegment[] {
+  if (sqlNode.type !== AST_NODE_TYPES.TemplateLiteral) {
+    // String literals: single segment, content starts one char after the opening quote.
+    return [{ sqlStart: 0, srcStart: sqlNode.range[0] + 1 }];
+  }
+  const rawSql = sqlNode.quasis
+    .map((quasi) => quasi.value.cooked ?? '')
+    .join('');
+  const trimStart = rawSql.length - rawSql.trimStart().length;
+  const segments: SqlSourceSegment[] = [];
+  let sqlCursor = 0;
+  for (const [index, quasi] of sqlNode.quasis.entries()) {
+    const cooked = quasi.value.cooked ?? '';
+    const localTrim = index === 0 ? trimStart : 0;
+    // quasi.range[0] is the opening backtick (index 0) or the closing } of the preceding expression.
+    segments.push({
+      sqlStart: sqlCursor,
+      srcStart: quasi.range[0] + 1 + localTrim,
+    });
+    sqlCursor += cooked.length - localTrim;
+  }
+  return segments;
+}
+
+function sqlOffsetToSource(
+  sqlOffset: number,
+  segments: SqlSourceSegment[],
+): number {
+  for (let index = segments.length - 1; index >= 0; index--) {
+    const seg = segments[index];
+    if (seg !== undefined && sqlOffset >= seg.sqlStart) {
+      return seg.srcStart + (sqlOffset - seg.sqlStart);
+    }
+  }
+  return segments[0]?.srcStart ?? 0;
+}
+
+// ---------------------------------------------------------------------------
+// ESLint rule
+// ---------------------------------------------------------------------------
+
+const rule: ESLintUtils.RuleModule<typeof SYNTAXT_ERROR | typeof ATHENA_ERROR> =
+  createRule({
+    name: ruleId,
+    meta: {
+      type: 'problem',
+      docs: {
+        description:
+          'Validate Athena SQL strings against OpenAPI schemas at lint time',
+      },
+      schema: [],
+      messages: {
+        [SYNTAXT_ERROR]: `SyntaxError {{ errorMessage }}`,
+        [ATHENA_ERROR]: `AthenaError {{ errorMessage }}`,
+      },
+    },
+    defaultOptions: [],
+    create(context) {
+      function checkSql(sql: string, sqlNode: TSESTree.Node) {
+        if (
+          !/^\s*(?:SELECT\b[\s\S]*\bFROM\b|WITH\b[\s\S]*\bSELECT\b[\s\S]*\b)/iu.test(
+            sql,
+          )
+        ) {
+          log('skipping non-SELECT SQL string', { sql });
+          return;
+        }
+
+        const sqlMapping = buildSqlMapping(sqlNode);
+        let ast: AST;
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+          ({ ast } = parse(sql, { includeLocations: true }));
+        } catch (error) {
+          log('error parsing Athena SQL', { error, sql });
+          const pegLoc = (
+            error as {
+              location?: { start: { offset: number }; end: { offset: number } };
+            }
+          ).location;
+          if (pegLoc !== undefined) {
+            const sourceText = context.sourceCode.getText();
+            context.report({
+              loc: {
+                start: offsetToLoc(
+                  sourceText,
+                  sqlOffsetToSource(pegLoc.start.offset, sqlMapping),
+                ),
+                end: offsetToLoc(
+                  sourceText,
+                  sqlOffsetToSource(pegLoc.end.offset, sqlMapping),
+                ),
+              },
+              messageId: SYNTAXT_ERROR,
+              data: { errorMessage: (error as Error).message },
+            });
+          } else {
+            context.report({
+              node: sqlNode,
+              messageId: SYNTAXT_ERROR,
+              data: { errorMessage: (error as Error).message },
+            });
+          }
+          return;
+        }
+        const athenaCtx = createRootContext();
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+          checkAthenaAst(Array.isArray(ast) ? ast[0] : ast, athenaCtx);
+        } catch (error) {
+          log('error checking Athena AST', { error, sql });
+          if (error instanceof AthenaError) {
+            const astLoc = (
+              error.ast as
+                | {
+                    loc?: {
+                      start: { offset: number };
+                      end: { offset: number };
+                    };
+                  }
+                | undefined
+            )?.loc;
+            if (astLoc !== undefined) {
+              const sourceText = context.sourceCode.getText();
+              context.report({
+                loc: {
+                  start: offsetToLoc(
+                    sourceText,
+                    sqlOffsetToSource(astLoc.start.offset, sqlMapping),
+                  ),
+                  end: offsetToLoc(
+                    sourceText,
+                    sqlOffsetToSource(astLoc.end.offset, sqlMapping),
+                  ),
+                },
+                messageId: ATHENA_ERROR,
+                data: { errorMessage: error.message },
+              });
+            } else {
+              context.report({
+                node: sqlNode,
+                messageId: ATHENA_ERROR,
+                data: { errorMessage: error.message },
+              });
+            }
+          } else {
+            // eslint-disable-next-line no-console
+            console.error(
+              `Failed to apply ${ruleId} rule for "${context.filename}":`,
+              error,
+            );
+            context.report({
+              node: sqlNode,
+              messageId: ATHENA_ERROR,
+              data: {
+                errorMessage:
+                  error instanceof Error
+                    ? String(error)
+                    : JSON.stringify(error, undefined, 2),
+              },
+            });
+          }
+        }
+      }
+
+      return {
+        TemplateLiteral(sqlNode) {
+          const sql = sqlNode.quasis
+            .map((quasi) => quasi.value.cooked)
+            .join('')
+            .trim();
+          checkSql(sql, sqlNode);
+        },
+        Literal(sqlNode) {
+          if (typeof sqlNode.value !== 'string') {
+            return;
+          }
+          checkSql(sqlNode.value, sqlNode);
+        },
+      };
+    },
+  });
+
+export default rule;

package/src/athena/column.ts

@@ -0,0 +1,4 @@
+// athena/column.ts
+
+// Re-exported for external consumers that previously imported ResolvedColumn from here.
+export type { ResolvedColumn } from './context';

package/src/athena/context.ts

@@ -0,0 +1,47 @@
+// athena/context.ts
+
+import type { OpenAPIV3_1 as v3 } from 'openapi-types';
+
+import type { ApiSchemas } from '../openapi/generate-schema';
+import type { MatchedOperation } from './api-matcher';
+
+export interface ResolvedColumn {
+  name: string;
+  schema: v3.SchemaObject;
+  ast?: object;
+}
+
+export interface ResolvedTable {
+  name?: string;
+  columns: Map<string, ResolvedColumn[]>;
+  apiOperation?: MatchedOperation[];
+}
+
+export interface VisitContext {
+  // Tables available in this scope, keyed by name. Each name maps to an array because
+  // multiple API operations can match a single service table (e.g. GET + POST on same path).
+  tables: Map<string, ResolvedTable[]>;
+  // alias → canonical table name
+  aliases: Map<string, string>;
+  // API schema disk-read cache, shared across the entire query to avoid re-reading files.
+  apiSchemas: Map<string, ApiSchemas[]>;
+  parent?: VisitContext;
+}
+
+export function createRootContext(): VisitContext {
+  return {
+    tables: new Map(),
+    aliases: new Map(),
+    apiSchemas: new Map(),
+  };
+}
+
+// Creates a child context that inherits CTE tables registered in the parent scope.
+export function createChildContext(parent: VisitContext): VisitContext {
+  return {
+    tables: new Map(parent.tables),
+    aliases: new Map(),
+    apiSchemas: parent.apiSchemas,
+    parent,
+  };
+}

package/src/athena/index.ts

@@ -0,0 +1,13 @@
+// athena/index.ts
+
+import type { OpenAPIV3_1 as v3 } from 'openapi-types';
+
+export interface ServiceEndpoint {
+  path: string;
+  method: 'GET' | 'POST' | 'PUT' | 'DELETE';
+  requestBody?: v3.SchemaObject;
+  responses: Record<string, v3.SchemaObject>;
+}
+
+// A service may expose multiple endpoints (e.g. /sample/v1, /sample/v2).
+export type Service = Record<string, ServiceEndpoint>;

package/src/athena/service-table.ts

@@ -0,0 +1,78 @@
+// athena/service-table.ts
+
+/*
+ * Copyright (c) 2021-2026 Check Digit, LLC
+ *
+ * This code is licensed under the MIT license (see LICENSE.txt for details).
+ */
+
+/**
+ * Builds a ResolvedTable for one matched API operation against an Athena service table.
+ * Each Athena service table exposes a fixed set of top-level columns representing the
+ * Kinesis stream record fields (method, url, requestbody, responsebody, etc.).
+ */
+
+import type { OpenAPIV3_1 as v3 } from 'openapi-types';
+import type { SchemaObject } from 'ajv/dist/2020';
+
+import type { MatchedOperation } from './api-matcher';
+import type { ResolvedColumn, ResolvedTable } from './context';
+
+const SCHEMA_STRING: SchemaObject = { type: 'string' };
+const SCHEMA_OBJECT: SchemaObject = { type: 'object' };
+
+function col(name: string, schema: v3.SchemaObject): ResolvedColumn {
+  return { name, schema };
+}
+
+function bodySchema(
+  envelope: SchemaObject,
+  field: 'body' | 'headers',
+): v3.SchemaObject {
+  return (
+    (envelope as Record<string, Record<string, v3.SchemaObject>>)[
+      'properties'
+    ]?.[field] ?? SCHEMA_OBJECT
+  );
+}
+
+/**
+ * Create one ResolvedTable per matched API operation.
+ * Multiple operations may match (e.g. GET + POST for the same service), so the
+ * caller receives an array and stores all of them under the same table name.
+ */
+export function buildServiceTables(
+  tableName: string,
+  operations: MatchedOperation[],
+): ResolvedTable[] {
+  return operations.map((operation) => ({
+    name: tableName,
+    apiOperation: [operation],
+    columns: new Map<string, ResolvedColumn[]>([
+      ['method', [col('method', SCHEMA_STRING)]],
+      ['started', [col('started', SCHEMA_STRING)]],
+      ['ended', [col('ended', SCHEMA_STRING)]],
+      ['url', [col('url', SCHEMA_STRING)]],
+      [
+        'requestbody',
+        [col('requestbody', bodySchema(operation.request, 'body'))],
+      ],
+      [
+        'requestheaders',
+        [col('requestheaders', bodySchema(operation.request, 'headers'))],
+      ],
+      ['responsestatus', [col('responsestatus', SCHEMA_STRING)]],
+      ['responsemessage', [col('responsemessage', SCHEMA_STRING)]],
+      ['responsetype', [col('responsetype', SCHEMA_STRING)]],
+      [
+        'responsebody',
+        [col('responsebody', bodySchema(operation.response, 'body'))],
+      ],
+      [
+        'responseheaders',
+        [col('responseheaders', bodySchema(operation.response, 'headers'))],
+      ],
+      ['partition_date', [col('partition_date', SCHEMA_STRING)]],
+    ]),
+  }));
+}