@sanity/ailf 0.1.0

package/config/airbyte/ai_literacy_framework.connector.yaml

@@ -0,0 +1,639 @@
+# Airbyte Declarative Source — AI Literacy Framework
+#
+# Extracts evaluation reports from the Sanity Content Lake and delivers them
+# to BigQuery (or any Airbyte-supported destination).
+#
+# Architecture: Sanity Content Lake → Airbyte (scheduled poll) → BigQuery
+# This replaces the direct BigQuerySink with ELT managed by the data team.
+#
+# Two streams:
+#   1. reports — one row per evaluation run, GROQ-projected to flat columns
+#   2. area_scores — one row per report with nested model×area scores;
+#      use BigQuery views (see bigquery/views/) to UNNEST into flat rows
+#
+# Both streams use incremental sync with _createdAt as the cursor, so only
+# new reports are transferred on each sync.
+#
+# @see docs/design-docs/report-store/bigquery.md — target schema
+# @see docs/design-docs/report-store/airbyte-elt.md — integration design
+version: 6.48.15
+
+type: DeclarativeSource
+
+check:
+  type: CheckStream
+  stream_names:
+    - reports
+
+definitions:
+  streams:
+    # ------------------------------------------------------------------
+    # Stream 1: reports — flat row per evaluation run
+    # ------------------------------------------------------------------
+    # GROQ projection flattens nested provenance/summary into top-level
+    # columns matching the ailf.reports BigQuery table schema.
+    # The comparison field is intentionally excluded — it duplicates
+    # entire ScoreSummary objects and can always be recomputed from
+    # two report rows.
+    reports:
+      type: DeclarativeStream
+      name: reports
+      retriever:
+        type: SimpleRetriever
+        decoder:
+          type: JsonDecoder
+        requester:
+          $ref: "#/definitions/base_requester"
+          path: /v2026-03-12/data/query/{{ config['dataset'] }}
+          http_method: GET
+          request_parameters:
+            query: >-
+              *[_type=="ailf.report" && _createdAt > "{{
+              stream_interval.start_time or '1970-01-01T00:00:00Z' }}" &&
+              _createdAt <= "{{ stream_interval.end_time }}" ]|order(_createdAt
+              asc){
+                "report_id": reportId,
+                "completed_at": completedAt,
+                "duration_ms": durationMs,
+                tag,
+                "mode": provenance.mode,
+                "source_name": provenance.source.name,
+                "source_base_url": provenance.source.baseUrl,
+                "source_dataset": provenance.source.dataset,
+                "source_perspective": provenance.source.perspective,
+                "grader_model": provenance.graderModel,
+                "trigger_type": provenance.trigger.type,
+                "trigger_caller_repo": select(
+                  provenance.trigger.type == "cross-repo" =>
+              provenance.trigger.callerRepo,
+                  null
+                ),
+                "git_repo": provenance.git.repo,
+                "git_branch": provenance.git.branch,
+                "git_sha": provenance.git.sha,
+                "git_pr_number": provenance.git.prNumber,
+                "avg_score": summary.overall.avgScore,
+                "avg_doc_lift": summary.overall.avgDocLift,
+                "total_cost": summary.overall.cost.total,
+                "grader_cost": summary.overall.cost.graderTotal,
+                "area_count": count(provenance.areas),
+                "model_count": count(provenance.models),
+                "areas": provenance.areas,
+                "models": provenance.models[].id,
+                "avg_actual_score": summary.overall.avgActualScore,
+                "avg_retrieval_gap": summary.overall.avgRetrievalGap,
+                "avg_infrastructure_efficiency":
+              summary.overall.avgInfrastructureEfficiency,
+                "promptfoo_url": provenance.promptfooUrl,
+                "promptfoo_urls": provenance.promptfooUrls[] { mode, url },
+                _createdAt
+              }
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path:
+              - result
+      primary_key:
+        - report_id
+      incremental_sync:
+        type: DatetimeBasedCursor
+        cursor_field: _createdAt
+        cursor_datetime_formats:
+          - "%Y-%m-%dT%H:%M:%S.%fZ"
+          - "%Y-%m-%dT%H:%M:%SZ"
+        datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+        start_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config.get('start_date', '2026-01-01T00:00:00Z') }}"
+          datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+        step: P30D
+        cursor_granularity: PT1S
+      schema_loader:
+        type: InlineSchemaLoader
+        schema:
+          $ref: "#/schemas/reports"
+
+    # ------------------------------------------------------------------
+    # Stream 2: area_scores — per-model per-area score rows
+    # ------------------------------------------------------------------
+    # GROQ extracts the nested perModel→scores arrays with report-level
+    # context (report_id, completed_at, mode, source_name). The nesting
+    # is preserved because GROQ cannot explode arrays into flat rows.
+    #
+    # BigQuery consumers should query the `ailf.area_scores` view
+    # (defined in bigquery/views/area_scores.sql) which UNNESTs the
+    # nested arrays into one flat row per area per model per report.
+    area_scores:
+      type: DeclarativeStream
+      name: area_scores
+      retriever:
+        type: SimpleRetriever
+        decoder:
+          type: JsonDecoder
+        requester:
+          $ref: "#/definitions/base_requester"
+          path: /v2026-03-12/data/query/{{ config['dataset'] }}
+          http_method: GET
+          request_parameters:
+            query: >-
+              *[_type=="ailf.report" && _createdAt > "{{
+              stream_interval.start_time or '1970-01-01T00:00:00Z' }}" &&
+              _createdAt <= "{{ stream_interval.end_time }}" ]|order(_createdAt
+              asc){
+                "report_id": reportId,
+                "completed_at": completedAt,
+                "mode": provenance.mode,
+                "source_name": provenance.source.name,
+                "model_scores": summary.perModel[]{
+                  "model_id": modelId,
+                  "areas": scores[]{
+                    "area": feature,
+                    "total_score": totalScore,
+                    "task_completion": taskCompletion,
+                    "code_correctness": codeCorrectness,
+                    "doc_coverage": docCoverage,
+                    "doc_lift": coalesce(docLift, liftFromDocs),
+                    "ceiling_score": coalesce(ceilingScore, withDocsScore),
+                    "floor_score": coalesce(floorScore, withoutDocsScore),
+                    "actual_score": actualScore,
+                    "retrieval_gap": retrievalGap,
+                    "infrastructure_efficiency": infrastructureEfficiency,
+                    "total_cost": totalCost,
+                    "test_count": testCount
+                  }
+                },
+                "fallback_model_id": provenance.models[0].id,
+                "fallback_scores": summary.scores[]{
+                  "area": feature,
+                  "total_score": totalScore,
+                  "task_completion": taskCompletion,
+                  "code_correctness": codeCorrectness,
+                  "doc_coverage": docCoverage,
+                  "doc_lift": coalesce(docLift, liftFromDocs),
+                  "ceiling_score": coalesce(ceilingScore, withDocsScore),
+                  "floor_score": coalesce(floorScore, withoutDocsScore),
+                  "actual_score": actualScore,
+                  "retrieval_gap": retrievalGap,
+                  "infrastructure_efficiency": infrastructureEfficiency,
+                  "total_cost": totalCost,
+                  "test_count": testCount
+                },
+                _createdAt
+              }
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path:
+              - result
+      primary_key:
+        - report_id
+      incremental_sync:
+        type: DatetimeBasedCursor
+        cursor_field: _createdAt
+        cursor_datetime_formats:
+          - "%Y-%m-%dT%H:%M:%S.%fZ"
+          - "%Y-%m-%dT%H:%M:%SZ"
+        datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+        start_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config.get('start_date', '2026-01-01T00:00:00Z') }}"
+          datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+        step: P30D
+        cursor_granularity: PT1S
+      schema_loader:
+        type: InlineSchemaLoader
+        schema:
+          $ref: "#/schemas/area_scores"
+
+  base_requester:
+    type: HttpRequester
+    url_base: https://{{ config['project_id'] }}.api.sanity.io
+    authenticator:
+      type: BearerAuthenticator
+      api_token: "{{ config['api_key'] }}"
+
+streams:
+  - $ref: "#/definitions/streams/reports"
+  - $ref: "#/definitions/streams/area_scores"
+
+spec:
+  type: Spec
+  connection_specification:
+    type: object
+    $schema: http://json-schema.org/draft-07/schema#
+    required:
+      - api_key
+      - dataset
+      - project_id
+    properties:
+      api_key:
+        type: string
+        order: 0
+        title: Sanity API Token
+        description: >-
+          A Sanity API token with read access to the dataset containing
+          ailf.report documents. Generate one at
+          https://www.sanity.io/manage/project/<project_id>/api#tokens
+        airbyte_secret: true
+      dataset:
+        type: string
+        order: 1
+        title: Dataset
+        description: >-
+          The Sanity dataset containing evaluation reports.
+        default: next
+      project_id:
+        type: string
+        order: 2
+        title: Project ID
+        description: >-
+          The Sanity project ID (e.g., "3do82whm").
+        default: 3do82whm
+      start_date:
+        type: string
+        order: 3
+        title: Start Date
+        description: >-
+          Only sync reports created after this date (ISO 8601). Defaults to
+          2026-01-01 if not set.
+        default: "2026-01-01T00:00:00Z"
+        examples:
+          - "2026-01-01T00:00:00Z"
+          - "2026-06-01T00:00:00Z"
+    additionalProperties: true
+
+metadata:
+  assist: {}
+  testedStreams:
+    reports:
+      hasRecords: true
+      streamHash: null
+      hasResponse: true
+      primaryKeysAreUnique: true
+      primaryKeysArePresent: true
+      responsesAreSuccessful: true
+    area_scores:
+      hasRecords: true
+      streamHash: null
+      hasResponse: true
+      primaryKeysAreUnique: true
+      primaryKeysArePresent: true
+      responsesAreSuccessful: true
+  autoImportSchema:
+    reports: false
+    area_scores: false
+
+# ======================================================================
+# Inline schemas — manually defined to match the designed BigQuery tables.
+# autoImportSchema is OFF so these don't drift with Sanity document changes.
+# ======================================================================
+
+schemas:
+  # ------------------------------------------------------------------
+  # reports schema — flat, matches ailf.reports BigQuery table
+  # ------------------------------------------------------------------
+  reports:
+    type: object
+    $schema: http://json-schema.org/schema#
+    required:
+      - report_id
+    properties:
+      report_id:
+        type: string
+        description: UUID v7 report identifier (primary key)
+      completed_at:
+        type:
+          - string
+          - "null"
+        description: ISO 8601 timestamp when the evaluation completed
+      duration_ms:
+        type:
+          - number
+          - "null"
+        description: Pipeline execution time in milliseconds
+      tag:
+        type:
+          - string
+          - "null"
+        description: Optional human-supplied label
+      mode:
+        type:
+          - string
+          - "null"
+        description: "Evaluation mode: baseline, observed, or agentic"
+      source_name:
+        type:
+          - string
+          - "null"
+        description: Documentation source name (e.g., "production")
+      source_base_url:
+        type:
+          - string
+          - "null"
+        description: Documentation source base URL
+      source_dataset:
+        type:
+          - string
+          - "null"
+        description: Sanity dataset used for evaluation
+      source_perspective:
+        type:
+          - string
+          - "null"
+        description: Sanity perspective (for content release evaluations)
+      grader_model:
+        type:
+          - string
+          - "null"
+        description: Model used for LLM grading
+      trigger_type:
+        type:
+          - string
+          - "null"
+        description:
+          "What triggered the evaluation: manual, ci, scheduled, webhook,
+          cross-repo"
+      trigger_caller_repo:
+        type:
+          - string
+          - "null"
+        description: Caller repository for cross-repo triggers
+      git_repo:
+        type:
+          - string
+          - "null"
+        description: Source repository (when run from CI)
+      git_branch:
+        type:
+          - string
+          - "null"
+        description: Source branch
+      git_sha:
+        type:
+          - string
+          - "null"
+        description: Commit SHA
+      git_pr_number:
+        type:
+          - number
+          - "null"
+        description: Pull request number (if applicable)
+      avg_score:
+        type:
+          - number
+          - "null"
+        description: Overall average AI literacy score (0–100)
+      avg_doc_lift:
+        type:
+          - number
+          - "null"
+        description: Overall documentation lift score
+      total_cost:
+        type:
+          - number
+          - "null"
+        description: Total evaluation cost in USD
+      grader_cost:
+        type:
+          - number
+          - "null"
+        description: Grader model cost in USD
+      area_count:
+        type:
+          - number
+          - "null"
+        description: Number of feature areas evaluated
+      model_count:
+        type:
+          - number
+          - "null"
+        description: Number of models evaluated
+      areas:
+        type:
+          - array
+          - "null"
+        items:
+          type: string
+        description: List of evaluated feature area names
+      models:
+        type:
+          - array
+          - "null"
+        items:
+          type: string
+        description: List of evaluated model IDs
+      avg_actual_score:
+        type:
+          - number
+          - "null"
+        description: Average score from agent-retrieved docs (full-mode only)
+      avg_retrieval_gap:
+        type:
+          - number
+          - "null"
+        description: Average ceiling minus actual across areas (full-mode only)
+      avg_infrastructure_efficiency:
+        type:
+          - number
+          - "null"
+        description: Average actual/ceiling ratio across areas (full-mode only)
+      promptfoo_url:
+        type:
+          - string
+          - "null"
+        description: Legacy single Promptfoo share URL
+      promptfoo_urls:
+        type:
+          - array
+          - "null"
+        description: Per-mode Promptfoo share URLs (one per sub-eval)
+        items:
+          type: object
+          properties:
+            mode:
+              type: string
+              description: "Evaluation mode: baseline, agentic, observed"
+            url:
+              type: string
+              description: Promptfoo share URL for this mode
+      _createdAt:
+        type:
+          - string
+          - "null"
+        description:
+          Sanity document creation timestamp (used as incremental cursor)
+    additionalProperties: true
+
+  # ------------------------------------------------------------------
+  # area_scores schema — nested model→area scores for BigQuery UNNEST
+  # ------------------------------------------------------------------
+  area_scores:
+    type: object
+    $schema: http://json-schema.org/schema#
+    required:
+      - report_id
+    properties:
+      report_id:
+        type: string
+        description: FK to reports table (UUID v7)
+      completed_at:
+        type:
+          - string
+          - "null"
+        description: Denormalized timestamp for partitioning
+      mode:
+        type:
+          - string
+          - "null"
+        description: Denormalized evaluation mode for clustering
+      source_name:
+        type:
+          - string
+          - "null"
+        description: Denormalized source name for clustering
+      model_scores:
+        type:
+          - array
+          - "null"
+        description:
+          Per-model score breakdowns (UNNEST in BigQuery to get flat rows)
+        items:
+          type: object
+          properties:
+            model_id:
+              type:
+                - string
+                - "null"
+            areas:
+              type:
+                - array
+                - "null"
+              items:
+                type: object
+                properties:
+                  area:
+                    type:
+                      - string
+                      - "null"
+                  total_score:
+                    type:
+                      - number
+                      - "null"
+                  task_completion:
+                    type:
+                      - number
+                      - "null"
+                  code_correctness:
+                    type:
+                      - number
+                      - "null"
+                  doc_coverage:
+                    type:
+                      - number
+                      - "null"
+                  doc_lift:
+                    type:
+                      - number
+                      - "null"
+                  ceiling_score:
+                    type:
+                      - number
+                      - "null"
+                  floor_score:
+                    type:
+                      - number
+                      - "null"
+                  total_cost:
+                    type:
+                      - number
+                      - "null"
+                  test_count:
+                    type:
+                      - number
+                      - "null"
+                  actual_score:
+                    type:
+                      - number
+                      - "null"
+                  retrieval_gap:
+                    type:
+                      - number
+                      - "null"
+                  infrastructure_efficiency:
+                    type:
+                      - number
+                      - "null"
+      fallback_model_id:
+        type:
+          - string
+          - "null"
+        description:
+          First model ID from provenance (used when perModel is absent)
+      fallback_scores:
+        type:
+          - array
+          - "null"
+        description: Aggregate scores without per-model breakdown (fallback)
+        items:
+          type: object
+          properties:
+            area:
+              type:
+                - string
+                - "null"
+            total_score:
+              type:
+                - number
+                - "null"
+            task_completion:
+              type:
+                - number
+                - "null"
+            code_correctness:
+              type:
+                - number
+                - "null"
+            doc_coverage:
+              type:
+                - number
+                - "null"
+            doc_lift:
+              type:
+                - number
+                - "null"
+            ceiling_score:
+              type:
+                - number
+                - "null"
+            floor_score:
+              type:
+                - number
+                - "null"
+            total_cost:
+              type:
+                - number
+                - "null"
+            test_count:
+              type:
+                - number
+                - "null"
+            actual_score:
+              type:
+                - number
+                - "null"
+            retrieval_gap:
+              type:
+                - number
+                - "null"
+            infrastructure_efficiency:
+              type:
+                - number
+                - "null"
+      _createdAt:
+        type:
+          - string
+          - "null"
+        description: Sanity document creation timestamp (incremental cursor)
+    additionalProperties: true

package/config/bigquery/README.md

@@ -0,0 +1,74 @@
+# BigQuery Schema & Views
+
+SQL definitions for the BigQuery analytics layer. These create the flattened
+tables and views that power SQL analytics, BI dashboards (Looker, Sheets), and
+ad-hoc data exploration.
+
+## Architecture
+
+```
+Sanity Content Lake
+       │
+       ▼
+Airbyte (scheduled sync)
+  ├─ Stream: reports    → ailf_raw.reports     (flat, GROQ-projected)
+  └─ Stream: area_scores → ailf_raw.area_scores (nested model→area arrays)
+       │
+       ▼
+BigQuery views (this directory)
+  ├─ ailf.reports       → direct passthrough (already flat from GROQ projection)
+  └─ ailf.area_scores   → UNNEST flattening (one row per area per model per report)
+```
+
+## Files
+
+| File                    | Purpose                                                                         |
+| ----------------------- | ------------------------------------------------------------------------------- |
+| `views/area_scores.sql` | Flattens nested `model_scores` array into one row per area per model per report |
+| `views/reports.sql`     | Clean passthrough view with correct types and column ordering                   |
+
+## Setup
+
+The Airbyte connection loads data into a raw dataset (e.g., `ailf_raw`). The
+views defined here reference that raw dataset and present the designed schema
+from `docs/design-docs/report-store/bigquery.md`.
+
+### 1. Create the raw dataset (Airbyte writes here)
+
+```bash
+bq mk --dataset data-platform-302218:ailf_raw
+```
+
+### 2. Create the analytics dataset (views live here)
+
+```bash
+bq mk --dataset data-platform-302218:ailf
+```
+
+### 3. Create the views
+
+```bash
+bq query --use_legacy_sql=false < views/reports.sql
+bq query --use_legacy_sql=false < views/area_scores.sql
+```
+
+## Naming conventions
+
+- **`ailf_raw.*`** — raw Airbyte-loaded tables (nested JSON, Airbyte metadata
+  columns)
+- **`ailf.*`** — analytics views (flat, typed, designed schema from bigquery.md)
+
+Airbyte adds metadata columns (`_airbyte_raw_id`, `_airbyte_extracted_at`,
+`_airbyte_meta`) to its output tables. The views strip these so downstream
+consumers see only the designed schema.
+
+## Schema evolution
+
+Views are the transformation layer. When the report format evolves:
+
+1. Update the GROQ projections in the Airbyte connector YAML
+2. Update the view SQL to map new fields
+3. Backfill is automatic — views always reflect current data
+
+@see docs/design-docs/report-store/bigquery.md — canonical schema definition
+@see packages/eval/config/airbyte/ — Airbyte connector configuration