@sanity/ailf 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -191,4 +191,4 @@ export interface ExampleRecord {
|
|
|
191
191
|
}
|
|
192
192
|
export declare const EXAMPLES: Record<ExampleType, ExampleRecord>;
|
|
193
193
|
/** GitHub Actions workflow template for AI Literacy evaluation */
|
|
194
|
-
export declare const workflowYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# AI Literacy Evaluation \u2014 GitHub Actions workflow\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This workflow submits evaluations to the AILF API when task or config\n# files change in a pull request. The API handles all processing\n# (LLM calls, doc fetching, grading, report publishing).\n#\n# Prerequisites:\n# Add one secret to your repository (Settings \u2192 Secrets \u2192 Actions):\n# AILF_API_KEY \u2014 your API key (starts with ailf_live_sk_)\n#\n# Customization:\n# - Adjust `paths` to match your documentation file locations\n# - Set full_eval to true for comprehensive (slower) evaluation\n# - See: https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/API_GATEWAY.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nname: AI Literacy Eval\n\non:\n pull_request:\n branches: [main]\n paths:\n - \".ailf/**\"\n\n # Manual trigger from the Actions tab\n workflow_dispatch:\n inputs:\n full_eval:\n description: \"Run full evaluation (all tests, slower)\"\n type: boolean\n default: false\n\nconcurrency:\n group: ailf-eval-${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\njobs:\n evaluate:\n name: AI Literacy Evaluation\n runs-on: ubuntu-latest\n permissions:\n pull-requests: write\n steps:\n # \u2500\u2500\u2500 Submit evaluation to the AILF API \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Submit evaluation\n id: submit\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n FULL_EVAL: ${{ inputs.full_eval || 'false' }}\n run: |\n if [ \"$FULL_EVAL\" = \"true\" ]; then\n DEBUG_FIELD=\"\"\n else\n DEBUG_FIELD='\"debug\": { \"enabled\": true, \"firstN\": 2 },'\n fi\n\n PAYLOAD=$(cat <<EOF\n {\n \"mode\": \"baseline\",\n ${DEBUG_FIELD}\n \"publish\": true,\n \"compare\": true\n }\n EOF\n )\n\n RESPONSE=$(curl -sf -X POST \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n https://ailf-api.sanity.build/v1/pipeline \\\n -d \"$PAYLOAD\")\n\n JOB_ID=$(echo \"$RESPONSE\" | jq -r '.jobId')\n echo \"job_id=$JOB_ID\" >> $GITHUB_OUTPUT\n echo \"\uD83D\uDCCB Submitted job: $JOB_ID\"\n\n # \u2500\u2500\u2500 Poll for results (long-polling) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Wait for results\n id: results\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n run: |\n for i in $(seq 1 40); do\n RESPONSE=$(curl -s \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Prefer: wait=25\" \\\n \"https://ailf-api.sanity.build/v1/jobs/$JOB_ID\")\n\n STATUS=$(echo \"$RESPONSE\" | jq -r '.status')\n\n case \"$STATUS\" in\n completed)\n echo \"status=completed\" >> $GITHUB_OUTPUT\n echo \"report_id=$(echo $RESPONSE | jq -r '.reportId // empty')\" >> $GITHUB_OUTPUT\n echo \"score=$(echo $RESPONSE | jq -r '.score // empty')\" >> $GITHUB_OUTPUT\n echo \"\u2705 Evaluation completed\"\n exit 0\n ;;\n failed|timed-out)\n echo \"status=$STATUS\" >> $GITHUB_OUTPUT\n echo \"::error::Evaluation $STATUS\"\n exit 1\n ;;\n *)\n echo \"\u23F3 [$i/40] $STATUS\"\n ;;\n esac\n done\n\n echo \"::error::Timed out waiting for evaluation\"\n exit 1\n\n # \u2500\u2500\u2500 Post results to PR \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Post PR comment\n if: >-\n always() && github.event_name == 'pull_request' &&\n steps.submit.outputs.job_id != ''\n uses: actions/github-script@v7\n env:\n JOB_STATUS: ${{ steps.results.outputs.status || 'unknown' }}\n REPORT_ID: ${{ steps.results.outputs.report_id || '' }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n SCORE: ${{ steps.results.outputs.score || '' }}\n with:\n script: |\n const marker = '<!-- ailf-score-report -->';\n const status = process.env.JOB_STATUS;\n const reportId = process.env.REPORT_ID;\n const jobId = process.env.JOB_ID;\n const score = process.env.SCORE;\n\n let icon, message;\n if (status === 'completed') {\n icon = '\u2705';\n message = score\n ? `Evaluation completed \u2014 score: **${score}/100**`\n : 'Evaluation completed successfully.';\n } else if (status === 'failed' || status === 'timed-out') {\n icon = '\u26A0\uFE0F';\n message = `Evaluation ${status}.`;\n } else {\n icon = '\u23F3';\n message = 'Evaluation status unknown (may still be running).';\n }\n\n let body = `${marker}\\n## ${icon} AI Literacy Evaluation\\n\\n${message}\\n`;\n if (reportId) {\n body += `\\n\uD83D\uDD17 [View detailed report](https://ailf-api.sanity.build/v1/reports/${reportId})\\n`;\n }\n body += `\\n<sub>Job: \\`${jobId}\\`</sub>\\n`;\n\n const { data: comments } = await github.rest.issues.listComments({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n });\n const existing = comments.find(c => c.body?.includes(marker));\n\n if (existing) {\n await github.rest.issues.updateComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: existing.id,\n body,\n });\n } else {\n await github.rest.issues.createComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n body,\n });\n }\n\n # \u2500\u2500\u2500 Job summary \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Summary\n if: always()\n env:\n JOB_STATUS: ${{ steps.results.outputs.status || 'unknown' }}\n REPORT_ID: ${{ steps.results.outputs.report_id || '' }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n SCORE: ${{ steps.results.outputs.score || '' }}\n run: |\n {\n echo \"## \uD83D\uDCCA AI Literacy Evaluation\"\n echo \"\"\n echo \"| Field | Value |\"\n echo \"|-------|-------|\"\n echo \"| Job | \\`$JOB_ID\\` |\"\n echo \"| Status | $JOB_STATUS |\"\n [ -n \"$SCORE\" ] && echo \"| Score | $SCORE/100 |\"\n [ -n \"$REPORT_ID\" ] && echo \"| Report | [$REPORT_ID](https://ailf-api.sanity.build/v1/reports/$REPORT_ID) |\"\n } >> \"$GITHUB_STEP_SUMMARY\"\n";
|
|
194
|
+
export declare const workflowYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# AI Literacy Evaluation \u2014 GitHub Actions workflow\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This workflow submits evaluations to the AILF API when task or config\n# files change in a pull request. The API handles all processing\n# (LLM calls, doc fetching, grading, report publishing).\n#\n# Prerequisites:\n# Add one secret to your repository (Settings \u2192 Secrets \u2192 Actions):\n# AILF_API_KEY \u2014 your API key (starts with ailf_live_sk_)\n#\n# Customization:\n# - Adjust `paths` to match your documentation file locations\n# - Set full_eval to true for comprehensive (slower) evaluation\n# - See: https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/API_GATEWAY.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nname: AI Literacy Eval\n\non:\n pull_request:\n branches: [main]\n paths:\n - \".ailf/**\"\n\n # Manual trigger from the Actions tab\n workflow_dispatch:\n inputs:\n full_eval:\n description: \"Run full evaluation (all tests, slower)\"\n type: boolean\n default: false\n\nconcurrency:\n group: ailf-eval-${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\njobs:\n evaluate:\n name: AI Literacy Evaluation\n runs-on: ubuntu-latest\n permissions:\n pull-requests: write\n steps:\n # \u2500\u2500\u2500 Submit evaluation to the AILF API \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Submit evaluation\n id: submit\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n FULL_EVAL: ${{ inputs.full_eval || 'false' }}\n run: |\n if [ \"$FULL_EVAL\" = \"true\" ]; then\n DEBUG_FIELD=\"\"\n else\n DEBUG_FIELD='\"debug\": { \"enabled\": true, \"firstN\": 2 },'\n fi\n\n PAYLOAD=$(cat <<EOF\n {\n \"mode\": \"baseline\",\n ${DEBUG_FIELD}\n \"publish\": true,\n \"compare\": true\n }\n EOF\n )\n\n RESPONSE=$(curl -sf -X POST \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n https://ailf-api.sanity.build/v1/pipeline \\\n -d \"$PAYLOAD\")\n\n JOB_ID=$(echo \"$RESPONSE\" | jq -r '.jobId')\n echo \"job_id=$JOB_ID\" >> $GITHUB_OUTPUT\n echo \"\uD83D\uDCCB Submitted job: $JOB_ID\"\n\n # \u2500\u2500\u2500 Poll for results (long-polling) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Wait for results\n id: results\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n run: |\n for i in $(seq 1 40); do\n RESPONSE=$(curl -s \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Prefer: wait=25\" \\\n \"https://ailf-api.sanity.build/v1/jobs/$JOB_ID\")\n\n STATUS=$(echo \"$RESPONSE\" | jq -r '.status')\n\n case \"$STATUS\" in\n completed)\n echo \"status=completed\" >> $GITHUB_OUTPUT\n echo \"report_id=$(echo $RESPONSE | jq -r '.reportId // empty')\" >> $GITHUB_OUTPUT\n echo \"score=$(echo $RESPONSE | jq -r '.score // empty')\" >> $GITHUB_OUTPUT\n echo \"\u2705 Evaluation completed\"\n exit 0\n ;;\n failed|timed-out)\n echo \"status=$STATUS\" >> $GITHUB_OUTPUT\n echo \"::error::Evaluation $STATUS\"\n exit 1\n ;;\n *)\n echo \"\u23F3 [$i/40] $STATUS\"\n ;;\n esac\n done\n\n echo \"::error::Timed out waiting for evaluation\"\n exit 1\n\n # \u2500\u2500\u2500 Fetch the markdown report \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Fetch markdown report\n id: markdown\n if: steps.results.outputs.report_id != ''\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n REPORT_ID: ${{ steps.results.outputs.report_id }}\n run: |\n REPORT_MD=$(curl -sf \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n \"https://ailf-api.sanity.build/v1/reports/$REPORT_ID/markdown\")\n\n if [ $? -eq 0 ] && [ -n \"$REPORT_MD\" ]; then\n # Write to a temp file to avoid shell quoting issues\n echo \"$REPORT_MD\" > /tmp/ailf-report.md\n echo \"fetched=true\" >> $GITHUB_OUTPUT\n else\n echo \"fetched=false\" >> $GITHUB_OUTPUT\n echo \"::warning::Could not fetch markdown report\"\n fi\n\n # \u2500\u2500\u2500 Post results to PR \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Post PR comment\n if: >-\n always() && github.event_name == 'pull_request' &&\n steps.submit.outputs.job_id != ''\n uses: actions/github-script@v7\n env:\n JOB_STATUS: ${{ steps.results.outputs.status || 'unknown' }}\n REPORT_ID: ${{ steps.results.outputs.report_id || '' }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n SCORE: ${{ steps.results.outputs.score || '' }}\n MARKDOWN_FETCHED: ${{ steps.markdown.outputs.fetched || 'false' }}\n with:\n script: |\n const fs = require('fs');\n const marker = '<!-- ailf-score-report -->';\n const status = process.env.JOB_STATUS;\n const reportId = process.env.REPORT_ID;\n const jobId = process.env.JOB_ID;\n const score = process.env.SCORE;\n const mdFetched = process.env.MARKDOWN_FETCHED === 'true';\n\n let body;\n\n if (status === 'completed' && mdFetched) {\n // Use the full markdown report from the API\n const reportMd = fs.readFileSync('/tmp/ailf-report.md', 'utf-8');\n body = `${marker}\\n${reportMd}`;\n if (reportId) {\n body += `\\n\uD83D\uDD17 [View in Studio](https://ailf-api.sanity.build/v1/reports/${reportId})`;\n }\n body += `\\n\\n<sub>Job: \\`${jobId}\\`</sub>\\n`;\n } else {\n // Fallback: minimal comment when markdown is unavailable\n let icon, message;\n if (status === 'completed') {\n icon = '\u2705';\n message = score\n ? `Evaluation completed \u2014 score: **${score}/100**`\n : 'Evaluation completed successfully.';\n } else if (status === 'failed' || status === 'timed-out') {\n icon = '\u26A0\uFE0F';\n message = `Evaluation ${status}.`;\n } else {\n icon = '\u23F3';\n message = 'Evaluation status unknown (may still be running).';\n }\n\n body = `${marker}\\n## ${icon} AI Literacy Evaluation\\n\\n${message}\\n`;\n if (reportId) {\n body += `\\n\uD83D\uDD17 [View detailed report](https://ailf-api.sanity.build/v1/reports/${reportId})\\n`;\n }\n body += `\\n<sub>Job: \\`${jobId}\\`</sub>\\n`;\n }\n\n const { data: comments } = await github.rest.issues.listComments({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n });\n const existing = comments.find(c => c.body?.includes(marker));\n\n if (existing) {\n await github.rest.issues.updateComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: existing.id,\n body,\n });\n } else {\n await github.rest.issues.createComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n body,\n });\n }\n\n # \u2500\u2500\u2500 Job summary \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n - name: Summary\n if: always()\n env:\n JOB_STATUS: ${{ steps.results.outputs.status || 'unknown' }}\n REPORT_ID: ${{ steps.results.outputs.report_id || '' }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n SCORE: ${{ steps.results.outputs.score || '' }}\n MARKDOWN_FETCHED: ${{ steps.markdown.outputs.fetched || 'false' }}\n run: |\n if [ \"$JOB_STATUS\" = \"completed\" ] && [ \"$MARKDOWN_FETCHED\" = \"true\" ] && [ -f /tmp/ailf-report.md ]; then\n # Use the full markdown report as the job summary\n cat /tmp/ailf-report.md >> \"$GITHUB_STEP_SUMMARY\"\n echo \"\" >> \"$GITHUB_STEP_SUMMARY\"\n [ -n \"$REPORT_ID\" ] && echo \"\uD83D\uDD17 [View in Studio](https://ailf-api.sanity.build/v1/reports/$REPORT_ID)\" >> \"$GITHUB_STEP_SUMMARY\"\n echo \"\" >> \"$GITHUB_STEP_SUMMARY\"\n echo \"<sub>Job: \\`$JOB_ID\\`</sub>\" >> \"$GITHUB_STEP_SUMMARY\"\n else\n # Fallback: minimal table\n {\n echo \"## \uD83D\uDCCA AI Literacy Evaluation\"\n echo \"\"\n echo \"| Field | Value |\"\n echo \"|-------|-------|\"\n echo \"| Job | \\`$JOB_ID\\` |\"\n echo \"| Status | $JOB_STATUS |\"\n [ -n \"$SCORE\" ] && echo \"| Score | $SCORE/100 |\"\n [ -n \"$REPORT_ID\" ] && echo \"| Report | [$REPORT_ID](https://ailf-api.sanity.build/v1/reports/$REPORT_ID) |\"\n } >> \"$GITHUB_STEP_SUMMARY\"\n fi\n";
|
|
@@ -289,4 +289,4 @@ export const EXAMPLES = {
|
|
|
289
289
|
// Raw file exports (non-data files, exported as raw strings)
|
|
290
290
|
// ---------------------------------------------------------------------------
|
|
291
291
|
/** GitHub Actions workflow template for AI Literacy evaluation */
|
|
292
|
-
export const workflowYaml = "# ──────────────────────────────────────────────────────────────────────\n# AI Literacy Evaluation — GitHub Actions workflow\n# ──────────────────────────────────────────────────────────────────────\n#\n# This workflow submits evaluations to the AILF API when task or config\n# files change in a pull request. The API handles all processing\n# (LLM calls, doc fetching, grading, report publishing).\n#\n# Prerequisites:\n# Add one secret to your repository (Settings → Secrets → Actions):\n# AILF_API_KEY — your API key (starts with ailf_live_sk_)\n#\n# Customization:\n# - Adjust `paths` to match your documentation file locations\n# - Set full_eval to true for comprehensive (slower) evaluation\n# - See: https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/API_GATEWAY.md\n# ──────────────────────────────────────────────────────────────────────\n\nname: AI Literacy Eval\n\non:\n pull_request:\n branches: [main]\n paths:\n - \".ailf/**\"\n\n # Manual trigger from the Actions tab\n workflow_dispatch:\n inputs:\n full_eval:\n description: \"Run full evaluation (all tests, slower)\"\n type: boolean\n default: false\n\nconcurrency:\n group: ailf-eval-${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\njobs:\n evaluate:\n name: AI Literacy Evaluation\n runs-on: ubuntu-latest\n permissions:\n pull-requests: write\n steps:\n # ─── Submit evaluation to the AILF API ─────────────────────\n - name: Submit evaluation\n id: submit\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n FULL_EVAL: ${{ inputs.full_eval || 'false' }}\n run: |\n if [ \"$FULL_EVAL\" = \"true\" ]; then\n DEBUG_FIELD=\"\"\n else\n DEBUG_FIELD='\"debug\": { \"enabled\": true, \"firstN\": 2 },'\n fi\n\n PAYLOAD=$(cat <<EOF\n {\n \"mode\": \"baseline\",\n ${DEBUG_FIELD}\n \"publish\": true,\n \"compare\": true\n }\n EOF\n )\n\n RESPONSE=$(curl -sf -X POST \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n https://ailf-api.sanity.build/v1/pipeline \\\n -d \"$PAYLOAD\")\n\n JOB_ID=$(echo \"$RESPONSE\" | jq -r '.jobId')\n echo \"job_id=$JOB_ID\" >> $GITHUB_OUTPUT\n echo \"📋 Submitted job: $JOB_ID\"\n\n # ─── Poll for results (long-polling) ───────────────────────\n - name: Wait for results\n id: results\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n run: |\n for i in $(seq 1 40); do\n RESPONSE=$(curl -s \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Prefer: wait=25\" \\\n \"https://ailf-api.sanity.build/v1/jobs/$JOB_ID\")\n\n STATUS=$(echo \"$RESPONSE\" | jq -r '.status')\n\n case \"$STATUS\" in\n completed)\n echo \"status=completed\" >> $GITHUB_OUTPUT\n echo \"report_id=$(echo $RESPONSE | jq -r '.reportId // empty')\" >> $GITHUB_OUTPUT\n echo \"score=$(echo $RESPONSE | jq -r '.score // empty')\" >> $GITHUB_OUTPUT\n echo \"✅ Evaluation completed\"\n exit 0\n ;;\n failed|timed-out)\n echo \"status=$STATUS\" >> $GITHUB_OUTPUT\n echo \"::error::Evaluation $STATUS\"\n exit 1\n ;;\n *)\n echo \"⏳ [$i/40] $STATUS\"\n ;;\n esac\n done\n\n echo \"::error::Timed out waiting for evaluation\"\n exit 1\n\n # ─── Post results to PR ────────────────────────────────────\n - name: Post PR comment\n if: >-\n always() && github.event_name == 'pull_request' &&\n steps.submit.outputs.job_id != ''\n uses: actions/github-script@v7\n env:\n JOB_STATUS: ${{ steps.results.outputs.status || 'unknown' }}\n REPORT_ID: ${{ steps.results.outputs.report_id || '' }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n SCORE: ${{ steps.results.outputs.score || '' }}\n with:\n script: |\n const marker = '<!-- ailf-score-report -->';\n const status = process.env.JOB_STATUS;\n const reportId = process.env.REPORT_ID;\n const jobId = process.env.JOB_ID;\n const score = process.env.SCORE;\n\n let icon, message;\n
|
|
292
|
+
export const workflowYaml = "# ──────────────────────────────────────────────────────────────────────\n# AI Literacy Evaluation — GitHub Actions workflow\n# ──────────────────────────────────────────────────────────────────────\n#\n# This workflow submits evaluations to the AILF API when task or config\n# files change in a pull request. The API handles all processing\n# (LLM calls, doc fetching, grading, report publishing).\n#\n# Prerequisites:\n# Add one secret to your repository (Settings → Secrets → Actions):\n# AILF_API_KEY — your API key (starts with ailf_live_sk_)\n#\n# Customization:\n# - Adjust `paths` to match your documentation file locations\n# - Set full_eval to true for comprehensive (slower) evaluation\n# - See: https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/API_GATEWAY.md\n# ──────────────────────────────────────────────────────────────────────\n\nname: AI Literacy Eval\n\non:\n pull_request:\n branches: [main]\n paths:\n - \".ailf/**\"\n\n # Manual trigger from the Actions tab\n workflow_dispatch:\n inputs:\n full_eval:\n description: \"Run full evaluation (all tests, slower)\"\n type: boolean\n default: false\n\nconcurrency:\n group: ailf-eval-${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\njobs:\n evaluate:\n name: AI Literacy Evaluation\n runs-on: ubuntu-latest\n permissions:\n pull-requests: write\n steps:\n # ─── Submit evaluation to the AILF API ─────────────────────\n - name: Submit evaluation\n id: submit\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n FULL_EVAL: ${{ inputs.full_eval || 'false' }}\n run: |\n if [ \"$FULL_EVAL\" = \"true\" ]; then\n DEBUG_FIELD=\"\"\n else\n DEBUG_FIELD='\"debug\": { \"enabled\": true, \"firstN\": 2 },'\n fi\n\n PAYLOAD=$(cat <<EOF\n {\n \"mode\": \"baseline\",\n ${DEBUG_FIELD}\n \"publish\": true,\n \"compare\": true\n }\n EOF\n )\n\n RESPONSE=$(curl -sf -X POST \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n https://ailf-api.sanity.build/v1/pipeline \\\n -d \"$PAYLOAD\")\n\n JOB_ID=$(echo \"$RESPONSE\" | jq -r '.jobId')\n echo \"job_id=$JOB_ID\" >> $GITHUB_OUTPUT\n echo \"📋 Submitted job: $JOB_ID\"\n\n # ─── Poll for results (long-polling) ───────────────────────\n - name: Wait for results\n id: results\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n run: |\n for i in $(seq 1 40); do\n RESPONSE=$(curl -s \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n -H \"Prefer: wait=25\" \\\n \"https://ailf-api.sanity.build/v1/jobs/$JOB_ID\")\n\n STATUS=$(echo \"$RESPONSE\" | jq -r '.status')\n\n case \"$STATUS\" in\n completed)\n echo \"status=completed\" >> $GITHUB_OUTPUT\n echo \"report_id=$(echo $RESPONSE | jq -r '.reportId // empty')\" >> $GITHUB_OUTPUT\n echo \"score=$(echo $RESPONSE | jq -r '.score // empty')\" >> $GITHUB_OUTPUT\n echo \"✅ Evaluation completed\"\n exit 0\n ;;\n failed|timed-out)\n echo \"status=$STATUS\" >> $GITHUB_OUTPUT\n echo \"::error::Evaluation $STATUS\"\n exit 1\n ;;\n *)\n echo \"⏳ [$i/40] $STATUS\"\n ;;\n esac\n done\n\n echo \"::error::Timed out waiting for evaluation\"\n exit 1\n\n # ─── Fetch the markdown report ─────────────────────────────\n - name: Fetch markdown report\n id: markdown\n if: steps.results.outputs.report_id != ''\n env:\n AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n REPORT_ID: ${{ steps.results.outputs.report_id }}\n run: |\n REPORT_MD=$(curl -sf \\\n -H \"Authorization: Bearer $AILF_API_KEY\" \\\n \"https://ailf-api.sanity.build/v1/reports/$REPORT_ID/markdown\")\n\n if [ $? -eq 0 ] && [ -n \"$REPORT_MD\" ]; then\n # Write to a temp file to avoid shell quoting issues\n echo \"$REPORT_MD\" > /tmp/ailf-report.md\n echo \"fetched=true\" >> $GITHUB_OUTPUT\n else\n echo \"fetched=false\" >> $GITHUB_OUTPUT\n echo \"::warning::Could not fetch markdown report\"\n fi\n\n # ─── Post results to PR ────────────────────────────────────\n - name: Post PR comment\n if: >-\n always() && github.event_name == 'pull_request' &&\n steps.submit.outputs.job_id != ''\n uses: actions/github-script@v7\n env:\n JOB_STATUS: ${{ steps.results.outputs.status || 'unknown' }}\n REPORT_ID: ${{ steps.results.outputs.report_id || '' }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n SCORE: ${{ steps.results.outputs.score || '' }}\n MARKDOWN_FETCHED: ${{ steps.markdown.outputs.fetched || 'false' }}\n with:\n script: |\n const fs = require('fs');\n const marker = '<!-- ailf-score-report -->';\n const status = process.env.JOB_STATUS;\n const reportId = process.env.REPORT_ID;\n const jobId = process.env.JOB_ID;\n const score = process.env.SCORE;\n const mdFetched = process.env.MARKDOWN_FETCHED === 'true';\n\n let body;\n\n if (status === 'completed' && mdFetched) {\n // Use the full markdown report from the API\n const reportMd = fs.readFileSync('/tmp/ailf-report.md', 'utf-8');\n body = `${marker}\\n${reportMd}`;\n if (reportId) {\n body += `\\n🔗 [View in Studio](https://ailf-api.sanity.build/v1/reports/${reportId})`;\n }\n body += `\\n\\n<sub>Job: \\`${jobId}\\`</sub>\\n`;\n } else {\n // Fallback: minimal comment when markdown is unavailable\n let icon, message;\n if (status === 'completed') {\n icon = '✅';\n message = score\n ? `Evaluation completed — score: **${score}/100**`\n : 'Evaluation completed successfully.';\n } else if (status === 'failed' || status === 'timed-out') {\n icon = '⚠️';\n message = `Evaluation ${status}.`;\n } else {\n icon = '⏳';\n message = 'Evaluation status unknown (may still be running).';\n }\n\n body = `${marker}\\n## ${icon} AI Literacy Evaluation\\n\\n${message}\\n`;\n if (reportId) {\n body += `\\n🔗 [View detailed report](https://ailf-api.sanity.build/v1/reports/${reportId})\\n`;\n }\n body += `\\n<sub>Job: \\`${jobId}\\`</sub>\\n`;\n }\n\n const { data: comments } = await github.rest.issues.listComments({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n });\n const existing = comments.find(c => c.body?.includes(marker));\n\n if (existing) {\n await github.rest.issues.updateComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: existing.id,\n body,\n });\n } else {\n await github.rest.issues.createComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n body,\n });\n }\n\n # ─── Job summary ───────────────────────────────────────────\n - name: Summary\n if: always()\n env:\n JOB_STATUS: ${{ steps.results.outputs.status || 'unknown' }}\n REPORT_ID: ${{ steps.results.outputs.report_id || '' }}\n JOB_ID: ${{ steps.submit.outputs.job_id }}\n SCORE: ${{ steps.results.outputs.score || '' }}\n MARKDOWN_FETCHED: ${{ steps.markdown.outputs.fetched || 'false' }}\n run: |\n if [ \"$JOB_STATUS\" = \"completed\" ] && [ \"$MARKDOWN_FETCHED\" = \"true\" ] && [ -f /tmp/ailf-report.md ]; then\n # Use the full markdown report as the job summary\n cat /tmp/ailf-report.md >> \"$GITHUB_STEP_SUMMARY\"\n echo \"\" >> \"$GITHUB_STEP_SUMMARY\"\n [ -n \"$REPORT_ID\" ] && echo \"🔗 [View in Studio](https://ailf-api.sanity.build/v1/reports/$REPORT_ID)\" >> \"$GITHUB_STEP_SUMMARY\"\n echo \"\" >> \"$GITHUB_STEP_SUMMARY\"\n echo \"<sub>Job: \\`$JOB_ID\\`</sub>\" >> \"$GITHUB_STEP_SUMMARY\"\n else\n # Fallback: minimal table\n {\n echo \"## 📊 AI Literacy Evaluation\"\n echo \"\"\n echo \"| Field | Value |\"\n echo \"|-------|-------|\"\n echo \"| Job | \\`$JOB_ID\\` |\"\n echo \"| Status | $JOB_STATUS |\"\n [ -n \"$SCORE\" ] && echo \"| Score | $SCORE/100 |\"\n [ -n \"$REPORT_ID\" ] && echo \"| Report | [$REPORT_ID](https://ailf-api.sanity.build/v1/reports/$REPORT_ID) |\"\n } >> \"$GITHUB_STEP_SUMMARY\"\n fi\n";
|