@sanity/ailf 0.1.28 → 0.1.29

package/dist/_vendor/ailf-core/ports/context.d.ts

@@ -35,6 +35,8 @@ export interface ResolvedConfig {
     areas?: string[];
     /** Task ID filter */
     tasks?: string[];
+    /** Tag filter — tasks must have at least one matching tag */
+    tags?: string[];
     /** Changed doc slugs for impact scoping */
     changedDocs?: string[];
     /** Documentation source name */

package/dist/_vendor/ailf-core/ports/task-source.d.ts

@@ -112,6 +112,8 @@ export interface TaskDefinition {
     baseline?: BaselineConfig;
     /** Additional template variables beyond task (e.g., custom vars) */
     extraVars?: Record<string, unknown>;
+    /** Freeform labels for filtering and organization */
+    tags?: string[];
 }
 /** Check if a canonical doc ref resolves by slug.
  *

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -179,6 +179,8 @@ export interface FeatureScore {
 export interface FilterOptions {
     /** Feature areas to include (filename stems, e.g., ["groq", "frameworks"]) */
     areas?: string[];
+    /** Tags to include — tasks must have at least one matching tag */
+    tags?: string[];
     /** Specific task IDs to include (e.g., ["groq-blog-queries"]) */
     taskIds?: string[];
 }

package/dist/adapters/api-client/build-request.d.ts

@@ -35,6 +35,7 @@ export interface RemoteConfigSlice {
     };
     areas?: string[];
     tasks?: string[];
+    tags?: string[];
     changedDocs?: string[];
     source?: string;
     compareEnabled?: boolean;

package/dist/adapters/api-client/build-request.js

@@ -167,12 +167,16 @@ function taskToInlineFormat(task) {
     if (task.baseline) {
         inline.baseline = task.baseline;
     }
+    if (task.tags?.length) {
+        inline.tags = task.tags;
+    }
     return inline;
 }
 function buildFilterOptions(config) {
     const areas = config.areas?.length ? config.areas : undefined;
     const taskIds = config.tasks?.length ? config.tasks : undefined;
-    if (!areas && !taskIds)
+    const tags = config.tags?.length ? config.tags : undefined;
+    if (!areas && !taskIds && !tags)
         return undefined;
-    return { areas, taskIds };
+    return { areas, taskIds, tags };
 }

package/dist/adapters/task-sources/content-lake-task-source.js

@@ -32,6 +32,7 @@ const TASKS_QUERY = /* groq */ `
   && (!defined($areas) || featureArea->areaId.current in $areas)
   && (!defined($taskIds) || id.current in $taskIds)
   && (execution.enabled != false)
+  && (!defined($tags) || count((tags)[@ in $tags]) > 0)
 ] | order(featureArea->areaId.current asc, id.current asc) {
   "taskId": id.current,
   description,
@@ -51,6 +52,7 @@ const TASKS_QUERY = /* groq */ `
   assert,
   rawAssert,
   baseline,
+  tags,
   "referenceSolutionTitle": referenceSolution->title
 }
 `;
@@ -90,6 +92,7 @@ function buildGroqParams(filter) {
         areas: filter?.areas && filter.areas.length > 0
             ? filter.areas.map((a) => a.toLowerCase())
             : null,
+        tags: filter?.tags && filter.tags.length > 0 ? filter.tags : null,
         taskIds: filter?.taskIds && filter.taskIds.length > 0 ? filter.taskIds : null,
     };
 }
@@ -116,6 +119,21 @@ function mapToTaskDefinition(raw) {
         .map(mapCanonicalDocRef)
         .filter((d) => d !== null);
     const assertions = mapAssertions(raw.assert ?? []);
+    // Append raw pass-through assertions (escape hatch for arbitrary Promptfoo
+    // assertion types that aren't in the curated list). These bypass template
+    // resolution and flow directly into the expanded Promptfoo test case as
+    // value-based assertions. In baseline mode, buildBaselineAsserts() with
+    // "abbreviated" (the default) drops non-rubric assertions, so rawAssert
+    // entries only run in the gold variant — consistent with how regular
+    // value-based assertions like `contains` or `regex` behave.
+    const rawAssertions = (raw.rawAssert ?? [])
+        .filter((a) => !!a.type)
+        .map((a) => ({
+        type: a.type,
+        ...(a.value !== undefined ? { value: a.value } : {}),
+        ...(a.threshold !== undefined ? { threshold: a.threshold } : {}),
+    }));
+    const allAssertions = [...assertions, ...rawAssertions];
     const baseline = raw.baseline
         ? {
             ...(raw.baseline.enabled !== undefined
@@ -129,7 +147,7 @@ function mapToTaskDefinition(raw) {
         }
         : undefined;
     return {
-        assertions,
+        assertions: allAssertions,
         canonicalDocs,
         description: raw.description,
         docCoverage: raw.docCoverage ?? false,
@@ -143,6 +161,7 @@ function mapToTaskDefinition(raw) {
         referenceSolution: "",
         taskPrompt: raw.taskPrompt,
         ...(baseline ? { baseline } : {}),
+        ...(raw.tags?.length ? { tags: raw.tags } : {}),
     };
 }
 /**

package/dist/adapters/task-sources/repo-task-source.js

@@ -79,6 +79,12 @@ export class RepoTaskSource {
                 if (entry.execution?.enabled === false) {
                     continue;
                 }
+                // Tag filter — skip tasks that don't match any requested tag
+                if (filter?.tags &&
+                    filter.tags.length > 0 &&
+                    (!entry.tags || !entry.tags.some((t) => filter.tags.includes(t)))) {
+                    continue;
+                }
                 definitions.push(mapToTaskDefinition(entry));
             }
         }
@@ -108,5 +114,6 @@ function mapToTaskDefinition(raw) {
         taskPrompt: typeof task === "string" ? task : "",
         ...(raw.baseline ? { baseline: raw.baseline } : {}),
         ...(extraVars ? { extraVars } : {}),
+        ...(raw.tags?.length ? { tags: raw.tags } : {}),
     };
 }

package/dist/commands/explain-handler.js

@@ -688,6 +688,7 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
         skipEval: raw.skipEval ?? false,
         skipFetch: raw.skipFetch ?? false,
         source: raw.source,
+        tag: raw.tag ?? [],
         task: raw.task,
         threshold: raw.threshold,
         url: raw.url ?? [],

package/dist/commands/pipeline-action.d.ts

@@ -54,6 +54,7 @@ export interface ResolvedOptions {
     remote: boolean;
     repoTasksPath?: string;
     taskOption?: string;
+    tagOption?: string[];
     taskSourceType?: "content-lake" | "repo" | "yaml";
     urlArgs: string[];
     apiUrl: string;

package/dist/commands/pipeline-action.js

@@ -126,6 +126,13 @@ export function computeResolvedOptions(opts) {
     // Scoping
     const areaOption = opts.area ?? process.env.EVAL_FILTER_AREAS ?? undefined;
     const taskOption = opts.task ?? process.env.EVAL_FILTER_TASKS ?? undefined;
+    const tagOption = opts.tag?.length
+        ? opts.tag
+        : process.env.EVAL_FILTER_TAGS
+            ? process.env.EVAL_FILTER_TAGS.split(",")
+                .map((s) => s.trim())
+                .filter(Boolean)
+            : undefined;
     const changedDocsOption = opts.changedDocs ?? process.env.EVAL_CHANGED_DOCS ?? undefined;
     // Document-driven scoping (pure — computes impactSummary without env writes)
     let impactSummary;
@@ -237,6 +244,7 @@ export function computeResolvedOptions(opts) {
             ? resolve(callerCwd, opts.repoTasksPath)
             : undefined,
         taskOption,
+        tagOption,
         taskSourceType: resolveTaskSourceType(opts.taskSource),
         urlArgs,
     };

package/dist/commands/pipeline.d.ts

@@ -56,6 +56,7 @@ export interface PipelineCliOptions {
     remote: boolean;
     repoTasksPath?: string;
     task?: string;
+    tag: string[];
     taskSource?: string;
     threshold?: number;
     url: string[];

package/dist/commands/pipeline.js

@@ -22,6 +22,13 @@ export function createPipelineCommand() {
         .option("--no-auto-scope", "Disable release-aware auto-scoping (evaluate all tasks even when a perspective is set)")
         .option("-a, --area <areas>", "Scope to feature areas (comma-separated)")
         .option("-t, --task <id>", "Scope to specific task ID")
+        .option("--tag <tags>", "Scope to tasks with matching tags (comma-separated, repeatable)", (val, prev) => [
+        ...prev,
+        ...val
+            .split(",")
+            .map((s) => s.trim())
+            .filter(Boolean),
+    ], [])
         .option("--changed-docs <slugs>", "Auto-scope to tasks affected by these document slugs")
         .option("-j, --concurrency <n>", "Max parallel API calls during evaluation", parseInt)
         .option("--grader-replications <n>", "Grader consistency replications", parseInt)

package/dist/orchestration/build-app-context.js

@@ -30,6 +30,7 @@ export function mapToResolvedConfig(opts, rootDir) {
             ?.split(",")
             .map((s) => s.trim())
             .filter(Boolean),
+        tags: opts.tagOption,
         changedDocs: opts.changedDocsOption
             ?.split(",")
             .map((s) => s.trim())

package/dist/orchestration/steps/fetch-docs-step.js

@@ -121,12 +121,13 @@ export class FetchDocsStep {
 // Helpers
 // ---------------------------------------------------------------------------
 function buildFilter(ctx) {
-    const { areas, tasks } = ctx.config;
-    if (!areas && !tasks)
+    const { areas, tasks, tags } = ctx.config;
+    if (!areas && !tasks && !tags)
         return undefined;
     return {
         ...(areas ? { areas } : {}),
         ...(tasks ? { taskIds: tasks } : {}),
+        ...(tags ? { tags } : {}),
     };
 }
 /**

package/dist/orchestration/steps/generate-configs-step.js

@@ -28,8 +28,12 @@ export class GenerateConfigsStep {
         // repo-based, and YAML tasks depending on which adapter is wired.
         let tasks;
         try {
-            const filter = ctx.config.areas || ctx.config.tasks
-                ? { areas: ctx.config.areas, taskIds: ctx.config.tasks }
+            const filter = ctx.config.areas || ctx.config.tasks || ctx.config.tags
+                ? {
+                    areas: ctx.config.areas,
+                    taskIds: ctx.config.tasks,
+                    tags: ctx.config.tags,
+                }
                 : undefined;
             tasks = await ctx.taskSource.loadTasks(filter);
         }
@@ -54,10 +58,11 @@ export class GenerateConfigsStep {
         try {
             generateConfigs({
                 allowedOrigins: ctx.config.allowedOrigins,
-                filter: ctx.config.areas || ctx.config.tasks
+                filter: ctx.config.areas || ctx.config.tasks || ctx.config.tags
                     ? {
                         areas: ctx.config.areas,
                         taskIds: ctx.config.tasks,
+                        tags: ctx.config.tags,
                     }
                     : undefined,
                 resolvedSource,

package/dist/orchestration/steps/run-eval-step.js

@@ -40,10 +40,11 @@ export class RunEvalStep {
         // Precondition: canonical context files exist for filtered tasks.
         // Must apply the same area/task filter as fetch-docs so we only
         // check contexts that were actually fetched.
-        const filter = ctx.config.areas || ctx.config.tasks
+        const filter = ctx.config.areas || ctx.config.tasks || ctx.config.tags
             ? {
                 ...(ctx.config.areas ? { areas: ctx.config.areas } : {}),
                 ...(ctx.config.tasks ? { taskIds: ctx.config.tasks } : {}),
+                ...(ctx.config.tags ? { tags: ctx.config.tags } : {}),
             }
             : undefined;
         let tasks = await ctx.taskSource.loadTasks(filter);
@@ -76,10 +77,11 @@ export class RunEvalStep {
         if (!debug?.enabled) {
             try {
                 evalFingerprint = computeEvalFingerprint({
-                    filter: ctx.config.areas || ctx.config.tasks
+                    filter: ctx.config.areas || ctx.config.tasks || ctx.config.tags
                         ? {
                             areas: ctx.config.areas,
                             taskIds: ctx.config.tasks,
+                            tags: ctx.config.tags,
                         }
                         : undefined,
                     graderModel: "default",

package/dist/pipeline/plan.d.ts

@@ -145,6 +145,7 @@ export interface PlanOptions {
     skipEval: boolean;
     skipFetch: boolean;
     source?: string;
+    tagOption?: string[];
     taskOption?: string;
 }
 /**

package/dist/pipeline/plan.js

@@ -117,7 +117,7 @@ export async function buildPipelinePlan(opts, rootDir) {
         .filter((i) => i.severity === "error")
         .map((i) => `[${i.source}] ${i.message}`);
     // 2. Expand tasks with filters
-    const filter = opts.areaOption || opts.taskOption
+    const filter = opts.areaOption || opts.taskOption || opts.tagOption?.length
         ? {
             areas: opts.areaOption
                 ? opts.areaOption.split(",").map((a) => a.trim())
@@ -125,6 +125,7 @@ export async function buildPipelinePlan(opts, rootDir) {
             taskIds: opts.taskOption
                 ? opts.taskOption.split(",").map((t) => t.trim())
                 : undefined,
+            tags: opts.tagOption,
         }
         : undefined;
     let totalTests = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.1.28",
+  "version": "0.1.29",
   "private": false,
   "publishConfig": {
     "access": "restricted"