@archal/cli 0.7.7 → 0.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -233,6 +233,7 @@ function parseCriterionLine(line, index) {
233
233
  } else {
234
234
  type = inferCriterionType(description);
235
235
  }
236
+ if (!description) return null;
236
237
  return {
237
238
  id: `criterion-${index + 1}`,
238
239
  description,
@@ -333,7 +334,11 @@ ${expectedBehavior}`.toLowerCase();
333
334
  github: ["github", "repository", "pull request", "create_issue", "create_pull_request", "merge_pull_request"],
334
335
  slack: ["slack", "slack channel", "send_message", "slack message", "direct message"],
335
336
  linear: ["linear", "linear ticket", "linear project", "linear cycle"],
336
- jira: ["jira", "jira sprint", "jira epic", "jira board"]
337
+ jira: ["jira", "jira sprint", "jira epic", "jira board"],
338
+ stripe: ["stripe", "payment", "refund", "subscription", "invoice", "charge"],
339
+ supabase: ["supabase", "database", "sql query", "database table"],
340
+ "google-workspace": ["google workspace", "gmail", "google calendar", "google drive", "google docs"],
341
+ browser: ["browser", "web page", "navigate to", "click on", "web content"]
337
342
  };
338
343
  for (const [twin, keywords] of Object.entries(twinKeywords)) {
339
344
  if (keywords.some((kw) => combined.includes(kw))) {
@@ -425,7 +430,9 @@ function validateScenario(scenario) {
425
430
  }
426
431
  }
427
432
  if (scenario.config.twins.length === 0) {
428
- errors.push("Scenario does not reference any known twins (specify in Config section or mention services in Setup/Expected Behavior)");
433
+ errors.push(
434
+ 'Scenario does not reference any known twins. Add a "## Config" section with "twins: github" (or slack, linear, jira, stripe, supabase, google-workspace, browser). Alternatively, mention the service name in ## Setup or ## Expected Behavior.'
435
+ );
429
436
  }
430
437
  if (scenario.config.timeout <= 0) {
431
438
  errors.push("Timeout must be a positive number");
@@ -2982,10 +2989,11 @@ var LlmApiError = class extends Error {
2982
2989
  };
2983
2990
  var RETRYABLE_STATUS_CODES2 = /* @__PURE__ */ new Set([429, 500, 502, 503, 529]);
2984
2991
  function detectProvider(model) {
2985
- if (model.startsWith("gemini-")) return "gemini";
2986
- if (model.startsWith("claude-")) return "anthropic";
2987
- if (model.startsWith("gpt-") || model.startsWith("o1-") || model.startsWith("o2-") || model.startsWith("o3-") || model.startsWith("o4-")) return "openai";
2988
- if (model.startsWith("llama") || model.startsWith("mixtral") || model.startsWith("mistral") || model.startsWith("deepseek") || model.startsWith("qwen") || model.startsWith("codestral") || model.startsWith("command")) return "openai-compatible";
2992
+ const normalized = model.toLowerCase();
2993
+ if (normalized.startsWith("gemini-")) return "gemini";
2994
+ if (normalized.startsWith("claude-") || normalized.startsWith("sonnet-") || normalized.startsWith("haiku-") || normalized.startsWith("opus-")) return "anthropic";
2995
+ if (normalized.startsWith("gpt-") || normalized.startsWith("o1-") || normalized.startsWith("o2-") || normalized.startsWith("o3-") || normalized.startsWith("o4-")) return "openai";
2996
+ if (normalized.startsWith("llama") || normalized.startsWith("mixtral") || normalized.startsWith("mistral") || normalized.startsWith("deepseek") || normalized.startsWith("qwen") || normalized.startsWith("codestral") || normalized.startsWith("command")) return "openai-compatible";
2989
2997
  return "openai-compatible";
2990
2998
  }
2991
2999
  var PROVIDER_ENV_VARS = {
@@ -3011,8 +3019,18 @@ function validateKeyForProvider(key, provider) {
3011
3019
  return void 0;
3012
3020
  }
3013
3021
  function resolveProviderApiKey(explicitKey, provider) {
3014
- if (explicitKey) return explicitKey;
3015
- return process.env[PROVIDER_ENV_VARS[provider]] ?? "";
3022
+ const normalizedExplicit = explicitKey.trim();
3023
+ const providerEnvKey = process.env[PROVIDER_ENV_VARS[provider]]?.trim() ?? "";
3024
+ if (!normalizedExplicit) return providerEnvKey;
3025
+ const mismatch = validateKeyForProvider(normalizedExplicit, provider);
3026
+ if (mismatch && providerEnvKey) {
3027
+ debug("Configured API key appears mismatched for provider; falling back to provider env key", {
3028
+ provider,
3029
+ providerEnvVar: PROVIDER_ENV_VARS[provider]
3030
+ });
3031
+ return providerEnvKey;
3032
+ }
3033
+ return normalizedExplicit;
3016
3034
  }
3017
3035
  var REQUEST_TIMEOUT_MS3 = 6e4;
3018
3036
  var MAX_RETRIES2 = 3;
@@ -3061,7 +3079,7 @@ async function callLlmViaArchal(options) {
3061
3079
  debug("Archal backend response", { model: actualModel, remaining: String(result.data.remaining ?? "unknown") });
3062
3080
  const isSeedGen = options.intent === "seed-generate";
3063
3081
  if (!modelMismatchWarned && !isSeedGen && options.model && actualModel && !actualModel.includes(options.model) && !options.model.includes(actualModel)) {
3064
- warn(`Requested model "${options.model}" but Archal backend used "${actualModel}". To use a specific model, set provider to "direct" with your own API key.`);
3082
+ debug(`Archal backend used "${actualModel}" (requested "${options.model}"). To use a specific model, set provider to "direct" with your own API key.`);
3065
3083
  modelMismatchWarned = true;
3066
3084
  }
3067
3085
  return result.data.text;
@@ -4182,8 +4200,35 @@ function filterByPredicate(items, predicate) {
4182
4200
  if (knownMatches.length > 0) {
4183
4201
  return { items: knownMatches, recognized: true };
4184
4202
  }
4203
+ const ACTION_VERBS = /* @__PURE__ */ new Set([
4204
+ "listed",
4205
+ "fetched",
4206
+ "retrieved",
4207
+ "found",
4208
+ "searched",
4209
+ "queried",
4210
+ "posted",
4211
+ "sent",
4212
+ "received",
4213
+ "notified",
4214
+ "alerted",
4215
+ "reviewed",
4216
+ "analyzed",
4217
+ "inspected",
4218
+ "checked",
4219
+ "verified",
4220
+ "triaged",
4221
+ "escalated",
4222
+ "assigned",
4223
+ "tagged",
4224
+ "labeled",
4225
+ "updated",
4226
+ "edited",
4227
+ "patched",
4228
+ "migrated"
4229
+ ]);
4185
4230
  const isSingleWord = !lowerPredicate.includes(" ");
4186
- if (isSingleWord) {
4231
+ if (isSingleWord && !ACTION_VERBS.has(lowerPredicate)) {
4187
4232
  const hasKnownField = items.some((item) => {
4188
4233
  if (typeof item !== "object" || item === null) return false;
4189
4234
  const obj = item;
@@ -5455,24 +5500,46 @@ ${JSON.stringify(context.stateDiff, null, 2)}
5455
5500
  ## Agent Trace Evidence
5456
5501
  ${traceEvidence}`;
5457
5502
  }
5503
+ function estimateTokens(value) {
5504
+ const json = JSON.stringify(value);
5505
+ return Math.ceil(json.length / 4);
5506
+ }
5507
+ var MAX_STATE_TOKENS = 4e4;
5458
5508
  function summarizeState(state) {
5459
5509
  const flat = flattenTwinState(state);
5460
5510
  const summary = {};
5461
5511
  for (const [key, value] of Object.entries(flat)) {
5462
5512
  if (Array.isArray(value)) {
5463
- if (value.length <= 100) {
5513
+ if (value.length <= 50) {
5464
5514
  summary[key] = value;
5465
5515
  } else {
5466
5516
  summary[key] = {
5467
5517
  _count: value.length,
5468
- _first20: value.slice(0, 20),
5469
- _last20: value.slice(-20)
5518
+ _first10: value.slice(0, 10),
5519
+ _last10: value.slice(-10)
5470
5520
  };
5471
5521
  }
5472
5522
  } else {
5473
5523
  summary[key] = value;
5474
5524
  }
5475
5525
  }
5526
+ let totalTokens = estimateTokens(summary);
5527
+ if (totalTokens > MAX_STATE_TOKENS) {
5528
+ const collectionSizes = Object.entries(summary).map(([key, value]) => ({ key, tokens: estimateTokens(value) })).sort((a, b) => b.tokens - a.tokens);
5529
+ for (const { key } of collectionSizes) {
5530
+ if (totalTokens <= MAX_STATE_TOKENS) break;
5531
+ const value = summary[key];
5532
+ if (!Array.isArray(value)) continue;
5533
+ const before = estimateTokens(value);
5534
+ summary[key] = {
5535
+ _count: value.length,
5536
+ _first5: value.slice(0, 5),
5537
+ _last5: value.slice(-5),
5538
+ _truncated: "Collection too large for evaluation \u2014 showing subset"
5539
+ };
5540
+ totalTokens -= before - estimateTokens(summary[key]);
5541
+ }
5542
+ }
5476
5543
  return summary;
5477
5544
  }
5478
5545
  function parseJudgeResponse(text) {
@@ -5572,6 +5639,15 @@ async function evaluateWithLlm(criterion, expectedBehavior, stateBefore, stateAf
5572
5639
  };
5573
5640
  }
5574
5641
  const message = err instanceof Error ? err.message : String(err);
5642
+ if (err instanceof LlmApiError && err.status === 400 && message.includes("too long")) {
5643
+ warn(`LLM judge prompt too large for criterion "${criterion.id}" \u2014 twin state may be too large for evaluation`);
5644
+ return {
5645
+ criterionId: criterion.id,
5646
+ status: "fail",
5647
+ confidence: 0,
5648
+ explanation: "LLM evaluation skipped: prompt exceeded model context window. The scenario state is too large for probabilistic evaluation. Consider using deterministic [D] criteria for this scenario."
5649
+ };
5650
+ }
5575
5651
  error(`LLM judge call failed: ${message}`);
5576
5652
  return {
5577
5653
  criterionId: criterion.id,
@@ -7933,7 +8009,11 @@ function mergeCollectionSchema(entitySchema, overrides) {
7933
8009
  if (override.default !== void 0) def.default = override.default;
7934
8010
  }
7935
8011
  if (isRequired) {
7936
- delete def.default;
8012
+ if (nullableSet.has(field)) {
8013
+ def.default = null;
8014
+ } else {
8015
+ delete def.default;
8016
+ }
7937
8017
  delete def.required;
7938
8018
  }
7939
8019
  merged[field] = def;
@@ -7951,6 +8031,9 @@ function buildSchemaFromOverrides(overrides) {
7951
8031
  const type = nullableSet.has(field) && !baseType.includes("null") && !baseType.includes("[]") ? `${baseType}|null` : baseType;
7952
8032
  schema[field] = {
7953
8033
  type,
8034
+ // If field is both required AND nullable, give it a null default so
8035
+ // the LLM can omit it without triggering a hard validation error.
8036
+ ...nullableSet.has(field) && { default: null },
7954
8037
  ...override?.aliases && { aliases: override.aliases },
7955
8038
  ...override?.description && { description: override.description },
7956
8039
  ...override?.fk && { fk: override.fk },
@@ -8063,6 +8146,8 @@ function validateSeedAgainstSchema(twinName, seed, baseEntityCounts) {
8063
8146
  errors.push(
8064
8147
  `${entityLabel}: wrong field name "${usedAlias}" - use "${fieldName}" instead`
8065
8148
  );
8149
+ } else if (def.type.includes("null")) {
8150
+ entity[fieldName] = null;
8066
8151
  } else {
8067
8152
  errors.push(
8068
8153
  `${entityLabel}: missing required field "${fieldName}" (${def.type})`
@@ -8220,7 +8305,8 @@ var RELATIONSHIP_RULES = {
8220
8305
  { sourceCollection: "disputes", sourceField: "paymentIntentId", targetCollection: "paymentIntents", targetField: "paymentIntentId", optional: true }
8221
8306
  ],
8222
8307
  jira: [
8223
- { sourceCollection: "issues", sourceField: "projectId", targetCollection: "projects", targetField: "id" }
8308
+ { sourceCollection: "issues", sourceField: "projectId", targetCollection: "projects", targetField: "id" },
8309
+ { sourceCollection: "projects", sourceField: "leadAccountId", targetCollection: "users", targetField: "accountId" }
8224
8310
  ],
8225
8311
  linear: [
8226
8312
  { sourceCollection: "issues", sourceField: "teamId", targetCollection: "teams", targetField: "id" },
@@ -8464,15 +8550,17 @@ function autoFillMissingFKs(seed, twinName) {
8464
8550
  const targetEntities = result[rule.targetCollection];
8465
8551
  if (!sourceEntities || !targetEntities || targetEntities.length === 0) continue;
8466
8552
  const targetValues = targetEntities.map((e) => e[rule.targetField]).filter((v) => v !== void 0 && v !== null);
8467
- if (targetValues.length !== 1) continue;
8468
- const singleTarget = targetValues[0];
8553
+ if (targetValues.length === 0) continue;
8554
+ let fillIndex = 0;
8469
8555
  for (const entity of sourceEntities) {
8470
8556
  const e = entity;
8471
8557
  if (e[rule.sourceField] === void 0 || e[rule.sourceField] === null) {
8472
- warn(
8473
- `Auto-filling ${rule.sourceCollection}.${rule.sourceField} = ${String(singleTarget)} (only one ${rule.targetCollection} exists)`
8558
+ const fillValue = targetValues[fillIndex % targetValues.length];
8559
+ fillIndex++;
8560
+ debug(
8561
+ `Auto-filling ${rule.sourceCollection}.${rule.sourceField} = ${String(fillValue)} (from ${targetValues.length} ${rule.targetCollection})`
8474
8562
  );
8475
- e[rule.sourceField] = singleTarget;
8563
+ e[rule.sourceField] = fillValue;
8476
8564
  }
8477
8565
  }
8478
8566
  }
@@ -8506,12 +8594,36 @@ function normalizeSeedData(seed, twinName) {
8506
8594
  }
8507
8595
  }
8508
8596
  }
8597
+ const collectionSchema = schema[collection];
8598
+ if (collectionSchema) {
8599
+ for (const [field, fieldDef] of Object.entries(collectionSchema)) {
8600
+ if (!(field in e) || e[field] === null || e[field] === void 0) continue;
8601
+ const expectedType = fieldDef.type.split("|")[0].trim();
8602
+ if (expectedType === "string" && typeof e[field] === "object" && e[field] !== null && !Array.isArray(e[field])) {
8603
+ const obj = e[field];
8604
+ const extracted = obj["login"] ?? obj["name"] ?? obj["value"] ?? obj["key"] ?? obj["id"] ?? obj["displayName"];
8605
+ if (typeof extracted === "string") {
8606
+ debug(`Seed normalization: coerced ${collection}.${field} from object to string "${extracted}"`);
8607
+ e[field] = extracted;
8608
+ } else {
8609
+ const firstStr = Object.values(obj).find((v) => typeof v === "string");
8610
+ if (firstStr) {
8611
+ debug(`Seed normalization: coerced ${collection}.${field} from object to string "${firstStr}" (fallback)`);
8612
+ e[field] = firstStr;
8613
+ } else {
8614
+ debug(`Seed normalization: could not coerce ${collection}.${field} from object to string, removing`);
8615
+ delete e[field];
8616
+ }
8617
+ }
8618
+ }
8619
+ }
8620
+ }
8509
8621
  if (collectionDefaults) {
8510
8622
  for (const [field, defaultValue] of Object.entries(collectionDefaults)) {
8511
8623
  if (!(field in e)) {
8512
8624
  e[field] = structuredClone(defaultValue);
8513
8625
  } else if (e[field] === null && defaultValue !== null) {
8514
- const fieldDef = schema[collection]?.[field];
8626
+ const fieldDef = collectionSchema?.[field];
8515
8627
  if (fieldDef && !fieldDef.type.includes("null")) {
8516
8628
  e[field] = structuredClone(defaultValue);
8517
8629
  }
@@ -8520,6 +8632,15 @@ function normalizeSeedData(seed, twinName) {
8520
8632
  }
8521
8633
  }
8522
8634
  }
8635
+ if (twinName === "github" && result["repos"]) {
8636
+ for (const entity of result["repos"]) {
8637
+ const e = entity;
8638
+ if ((!e["fullName"] || typeof e["fullName"] !== "string") && typeof e["owner"] === "string" && typeof e["name"] === "string") {
8639
+ e["fullName"] = `${e["owner"]}/${e["name"]}`;
8640
+ debug(`Seed normalization: derived repos.fullName = "${e["fullName"]}"`);
8641
+ }
8642
+ }
8643
+ }
8523
8644
  return result;
8524
8645
  }
8525
8646
 
@@ -8537,7 +8658,6 @@ var KIND_COLLECTION_HINTS = {
8537
8658
  event: ["events"],
8538
8659
  email: ["gmail_messages", "messages"]
8539
8660
  };
8540
- var STRICT_QUOTE_TWINS = /* @__PURE__ */ new Set(["slack", "google-workspace"]);
8541
8661
  var ENTITY_KEY_ALIASES = {
8542
8662
  "repo.owner": ["ownerLogin", "owner_login", "login", "owner.login", "owner.name"],
8543
8663
  "issue.key": ["identifier"],
@@ -8696,13 +8816,20 @@ function validateSeedCoverage(intent, mergedSeed) {
8696
8816
  const entityIssues = [];
8697
8817
  const quoteErrors = [];
8698
8818
  const quoteWarnings = [];
8819
+ const CORE_ENTITY_KEYS = /* @__PURE__ */ new Set(["owner", "name", "fullName", "channel_name", "key", "identifier", "number"]);
8820
+ const entityWarnings = [];
8699
8821
  for (const entity of intent.entities) {
8700
8822
  if (typeof entity.value === "boolean") continue;
8701
8823
  if (!valueExistsInCollections(mergedSeed, entity.kind, entity.key, entity.value)) {
8702
- entityIssues.push({
8824
+ const issue = {
8703
8825
  type: "missing_entity",
8704
8826
  message: `Expected ${entity.kind}.${entity.key}=${String(entity.value)} to exist`
8705
- });
8827
+ };
8828
+ if (CORE_ENTITY_KEYS.has(entity.key)) {
8829
+ entityIssues.push(issue);
8830
+ } else {
8831
+ entityWarnings.push(issue);
8832
+ }
8706
8833
  }
8707
8834
  }
8708
8835
  for (const quote of intent.quotedStrings) {
@@ -8715,18 +8842,14 @@ function validateSeedCoverage(intent, mergedSeed) {
8715
8842
  type: "missing_quote",
8716
8843
  message: `Expected quoted text to exist: "${quote}"`
8717
8844
  };
8718
- if (STRICT_QUOTE_TWINS.has(intent.twinName)) {
8719
- quoteErrors.push(issue);
8720
- } else {
8721
- quoteWarnings.push(issue);
8722
- }
8845
+ quoteWarnings.push(issue);
8723
8846
  }
8724
8847
  }
8725
8848
  const errors = [...entityIssues, ...quoteErrors];
8726
8849
  return {
8727
8850
  valid: errors.length === 0,
8728
8851
  issues: errors,
8729
- warnings: quoteWarnings
8852
+ warnings: [...quoteWarnings, ...entityWarnings]
8730
8853
  };
8731
8854
  }
8732
8855
 
@@ -8794,7 +8917,24 @@ var NON_SUBJECT_STARTS = /* @__PURE__ */ new Set([
8794
8917
  "could",
8795
8918
  "would",
8796
8919
  "may",
8797
- "might"
8920
+ "might",
8921
+ "for",
8922
+ "with",
8923
+ "in",
8924
+ "at",
8925
+ "to",
8926
+ "from",
8927
+ "by",
8928
+ "on",
8929
+ "per",
8930
+ "via",
8931
+ "into",
8932
+ "onto",
8933
+ "over",
8934
+ "under",
8935
+ "after",
8936
+ "before",
8937
+ "during"
8798
8938
  ]);
8799
8939
  function isReasonableCountSubject(subject, expected) {
8800
8940
  if (expected > MAX_REASONABLE_COUNT) return false;
@@ -8805,6 +8945,10 @@ function isReasonableCountSubject(subject, expected) {
8805
8945
  if (/\b(?:have|has|had|were|was|are|is|been|being|do|does|did|can|could|should|will|would|may|might)\b/.test(subject.toLowerCase())) return false;
8806
8946
  return true;
8807
8947
  }
8948
+ function appearsToBeClockSuffix(text, numberStart) {
8949
+ const prefix = text.slice(Math.max(0, numberStart - 3), numberStart);
8950
+ return /^\d{1,2}:$/.test(prefix);
8951
+ }
8808
8952
  function verifySeedCounts(setupText, seedState) {
8809
8953
  const mismatches = [];
8810
8954
  const flat = flattenTwinState(seedState);
@@ -8812,6 +8956,7 @@ function verifySeedCounts(setupText, seedState) {
8812
8956
  for (const match of setupText.matchAll(countPattern)) {
8813
8957
  const expected = parseInt(match[1], 10);
8814
8958
  const subject = match[2].trim();
8959
+ if (match.index !== void 0 && appearsToBeClockSuffix(setupText, match.index)) continue;
8815
8960
  if (!subject || expected <= 0) continue;
8816
8961
  if (!isReasonableCountSubject(subject, expected)) continue;
8817
8962
  const resolved = resolveSubjectInState(subject, flat);
@@ -8824,6 +8969,7 @@ function verifySeedCounts(setupText, seedState) {
8824
8969
  for (const match of setupText.matchAll(simplePattern)) {
8825
8970
  const expected = parseInt(match[1], 10);
8826
8971
  const subject = match[2].trim();
8972
+ if (match.index !== void 0 && appearsToBeClockSuffix(setupText, match.index)) continue;
8827
8973
  if (!subject || expected <= 0 || seenSubjects.has(subject.toLowerCase())) continue;
8828
8974
  if (!isReasonableCountSubject(subject, expected)) continue;
8829
8975
  const resolved = resolveSubjectInState(subject, flat);
@@ -9039,14 +9185,955 @@ function cacheNegativeSeed(twinName, baseSeedName, setupText, missingSlots, scop
9039
9185
  }
9040
9186
  }
9041
9187
 
9188
+ // src/runner/seed-blueprint.ts
9189
+ var BLUEPRINT_SYSTEM_PROMPT = `You are a precise data extraction tool. Given a scenario setup description, extract a structured JSON blueprint describing what test data needs to be created.
9190
+
9191
+ Output ONLY valid JSON matching this schema:
9192
+ {
9193
+ "identities": [
9194
+ { "collection": "<collection_name>", "fields": { "<field>": "<value>", ... } }
9195
+ ],
9196
+ "collections": [
9197
+ {
9198
+ "name": "<collection_name>",
9199
+ "totalCount": <number>,
9200
+ "groups": [
9201
+ {
9202
+ "count": <number>,
9203
+ "properties": { "<property>": <value>, ... }
9204
+ }
9205
+ ],
9206
+ "contentHint": "<optional description of what content should look like>"
9207
+ }
9208
+ ]
9209
+ }
9210
+
9211
+ Rules:
9212
+ - "identities" are named entities that MUST exist exactly (repos, channels, users with specific names)
9213
+ - "collections" describe bulk entities with count distributions
9214
+ - Group counts MUST sum to totalCount exactly
9215
+ - Use these property names for temporal attributes:
9216
+ - "stale": true/false (not updated in 90+ days)
9217
+ - "recentlyActive": true/false (updated within 30 days)
9218
+ - Use "state" for entity state: "open", "closed", "merged", etc.
9219
+ - Use "labels" for label arrays: ["bug", "keep-open"]
9220
+ - Use "assigned" for assignment: true/false or assignee name
9221
+ - Use "hasComments": true/false for whether entity has comments
9222
+ - Use "priority" for priority: "P0", "P1", "P2", "critical", "high", "medium", "low"
9223
+ - Use "private" for visibility: true/false
9224
+ - Keep it simple \u2014 only include what the setup text explicitly states
9225
+ - If the setup says "N of those" or "N of the X", that's a subset of the previous group
9226
+ - Do NOT include fields not mentioned in the setup text`;
9227
+ async function extractBlueprint(setupText, twinName, availableCollections, config) {
9228
+ const userPrompt = `Twin: ${twinName}
9229
+ Available collections: ${availableCollections.join(", ")}
9230
+
9231
+ Setup text:
9232
+ ${setupText}
9233
+
9234
+ Extract the seed blueprint as JSON.`;
9235
+ try {
9236
+ const provider = detectProvider(config.model);
9237
+ const apiKey = resolveProviderApiKey(config.apiKey, provider);
9238
+ const responseText = await callLlm({
9239
+ provider,
9240
+ model: config.model,
9241
+ apiKey,
9242
+ systemPrompt: BLUEPRINT_SYSTEM_PROMPT,
9243
+ userPrompt,
9244
+ maxTokens: 4096,
9245
+ baseUrl: config.baseUrl,
9246
+ providerMode: config.providerMode,
9247
+ intent: "seed-generate",
9248
+ responseFormat: "json"
9249
+ });
9250
+ if (!responseText) {
9251
+ warn("Blueprint extraction returned no text");
9252
+ return null;
9253
+ }
9254
+ const parsed = parseBlueprint(responseText, twinName);
9255
+ if (!parsed) return null;
9256
+ const validCollections = new Set(availableCollections);
9257
+ parsed.collections = parsed.collections.filter((col) => {
9258
+ if (validCollections.has(col.name)) return true;
9259
+ warn(`Blueprint references unknown collection "${col.name}" for ${twinName} \u2014 dropping`);
9260
+ return false;
9261
+ });
9262
+ for (const col of parsed.collections) {
9263
+ const groupSum = col.groups.reduce((sum, g) => sum + g.count, 0);
9264
+ if (groupSum !== col.totalCount) {
9265
+ debug(`Blueprint group count mismatch for ${col.name}: groups sum to ${groupSum}, totalCount is ${col.totalCount}. Adjusting.`);
9266
+ col.totalCount = groupSum;
9267
+ }
9268
+ if (col.totalCount === 0) {
9269
+ debug(`Blueprint collection ${col.name} has 0 entities \u2014 dropping`);
9270
+ }
9271
+ }
9272
+ parsed.collections = parsed.collections.filter((col) => col.totalCount > 0);
9273
+ if (parsed.collections.length === 0 && parsed.identities.length === 0) {
9274
+ warn("Blueprint extracted no valid collections or identities");
9275
+ return null;
9276
+ }
9277
+ return parsed;
9278
+ } catch (err) {
9279
+ const msg = err instanceof Error ? err.message : String(err);
9280
+ warn(`Blueprint extraction failed: ${msg}`);
9281
+ return null;
9282
+ }
9283
+ }
9284
+ function parseBlueprint(text, twinName) {
9285
+ try {
9286
+ let cleaned = text.trim();
9287
+ if (cleaned.startsWith("```")) {
9288
+ cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
9289
+ }
9290
+ const raw = JSON.parse(cleaned);
9291
+ if (!raw || typeof raw !== "object") return null;
9292
+ const identities = [];
9293
+ for (const id of raw.identities ?? []) {
9294
+ if (id?.collection && typeof id.fields === "object") {
9295
+ identities.push({ collection: String(id.collection), fields: id.fields });
9296
+ }
9297
+ }
9298
+ const collections = [];
9299
+ for (const col of raw.collections ?? []) {
9300
+ if (!col?.name || typeof col.totalCount !== "number") continue;
9301
+ const groups = [];
9302
+ for (const g of col.groups ?? []) {
9303
+ if (typeof g?.count === "number" && g.count > 0) {
9304
+ groups.push({ count: g.count, properties: g.properties ?? {} });
9305
+ }
9306
+ }
9307
+ if (groups.length === 0) {
9308
+ groups.push({ count: col.totalCount, properties: {} });
9309
+ }
9310
+ collections.push({
9311
+ name: String(col.name),
9312
+ totalCount: col.totalCount,
9313
+ groups,
9314
+ contentHint: col.contentHint
9315
+ });
9316
+ }
9317
+ return { twin: twinName, identities, collections };
9318
+ } catch {
9319
+ warn("Failed to parse blueprint JSON");
9320
+ return null;
9321
+ }
9322
+ }
9323
+
9324
+ // src/runner/seed-builder.ts
9325
+ var MS_PER_DAY = 864e5;
9326
+ function daysAgo(days, base) {
9327
+ const now = base ?? /* @__PURE__ */ new Date();
9328
+ return new Date(now.getTime() - days * MS_PER_DAY).toISOString();
9329
+ }
9330
+ function recentDate(withinDays = 7, base) {
9331
+ const now = base ?? /* @__PURE__ */ new Date();
9332
+ const offset = Math.floor(Math.random() * withinDays) * MS_PER_DAY;
9333
+ return new Date(now.getTime() - offset).toISOString();
9334
+ }
9335
+ function staleDate(minDays = 91, maxDays = 300, base) {
9336
+ const now = base ?? /* @__PURE__ */ new Date();
9337
+ const range = maxDays - minDays;
9338
+ const offset = (minDays + Math.floor(Math.random() * range)) * MS_PER_DAY;
9339
+ return new Date(now.getTime() - offset).toISOString();
9340
+ }
9341
+ var ISSUE_TITLES = [
9342
+ "Fix login redirect loop on Safari",
9343
+ "Add dark mode support to settings page",
9344
+ "Rate limiter returns 500 instead of 429",
9345
+ "Memory leak in WebSocket connection manager",
9346
+ "Dropdown menus do not close on mobile",
9347
+ "Search results do not highlight matching terms",
9348
+ "File upload fails silently for large files",
9349
+ "Add CSV export to analytics dashboard",
9350
+ "Implement bulk actions for notifications",
9351
+ "GraphQL query returns stale cached data",
9352
+ "Table component lacks keyboard navigation",
9353
+ "Create onboarding tutorial for new members",
9354
+ "API response times degrade for large date ranges",
9355
+ "Add two-factor authentication for admin accounts",
9356
+ "Deprecated crypto API warnings in test suite",
9357
+ "Evaluate replacing Moment.js with date-fns",
9358
+ "Broken tooltip positioning on Safari 17",
9359
+ "Login page shows blank screen on slow 3G",
9360
+ "Intermittent 502 errors on reports endpoint",
9361
+ "Tracking: migrate auth from cookies to JWT",
9362
+ "Roadmap: accessibility audit and WCAG compliance",
9363
+ "Upgrade Node.js runtime from 18 to 20 LTS",
9364
+ "Refactor database connection pool configuration",
9365
+ "Add retry logic for third-party API calls",
9366
+ "Improve error messages for form validation",
9367
+ "Pagination breaks when filtering by status",
9368
+ "Email notification templates are not responsive",
9369
+ "CI pipeline takes 45 minutes on large PRs",
9370
+ "Localization support for date and number formats",
9371
+ "Dashboard widgets do not resize on mobile"
9372
+ ];
9373
+ var ISSUE_BODIES = [
9374
+ "This has been reported multiple times by users. Needs investigation and a fix.",
9375
+ "No one has started working on this yet. Low priority but would improve UX.",
9376
+ "This is affecting production traffic. We need to address this soon.",
9377
+ "Users have requested this feature multiple times. Would be a nice enhancement.",
9378
+ "This is a long-running tracking issue. Should remain open for visibility.",
9379
+ "Repro steps: 1) Open the app 2) Navigate to settings 3) Observe the issue.",
9380
+ "This blocks other work. Please prioritize.",
9381
+ "Nice to have. Not urgent but would reduce tech debt."
9382
+ ];
9383
+ var PR_TITLES = [
9384
+ "Fix null check in JWT validation",
9385
+ "Add pagination to search results",
9386
+ "Refactor database connection pooling",
9387
+ "Update dependencies to latest versions",
9388
+ "Improve error handling in API middleware",
9389
+ "Add unit tests for auth module",
9390
+ "Fix race condition in WebSocket handler",
9391
+ "Migrate CSS to custom properties for theming"
9392
+ ];
9393
+ var CHANNEL_PURPOSES = [
9394
+ "General discussion",
9395
+ "Engineering team updates",
9396
+ "Product announcements",
9397
+ "Customer support escalations",
9398
+ "Random fun stuff",
9399
+ "Incident response",
9400
+ "Design feedback",
9401
+ "Deploy notifications"
9402
+ ];
9403
+ var MESSAGE_TEXTS = [
9404
+ "Hey team, just a heads up about the deploy today.",
9405
+ "Can someone review my PR? It's been open for a while.",
9406
+ "The CI pipeline is green again after the fix.",
9407
+ "Meeting notes from today's standup are in the doc.",
9408
+ "Has anyone seen this error before? Getting a 502 intermittently.",
9409
+ "Thanks for the quick turnaround on that bug fix!",
9410
+ "Reminder: retro is at 3pm today.",
9411
+ "I pushed a hotfix for the login issue. Please verify."
9412
+ ];
9413
+ function generateNodeId(prefix, id) {
9414
+ return `${prefix}_kgDOB${String(id).padStart(5, "0")}`;
9415
+ }
9416
+ function generateSha() {
9417
+ const hex = "0123456789abcdef";
9418
+ let sha = "";
9419
+ for (let i = 0; i < 40; i++) sha += hex[Math.floor(Math.random() * 16)];
9420
+ return sha;
9421
+ }
9422
+ function generateSlackId(prefix, index) {
9423
+ const base = "ABCDEFG0123456789";
9424
+ let id = prefix;
9425
+ let n = index + 100;
9426
+ for (let i = 0; i < 8; i++) {
9427
+ id += base[n % base.length];
9428
+ n = Math.floor(n / base.length) + i + 1;
9429
+ }
9430
+ return id;
9431
+ }
9432
+ function generateSlackTs(baseTs, index) {
9433
+ return `${baseTs + index * 60}.${String(100001 + index * 100).padStart(6, "0")}`;
9434
+ }
9435
+ function generateUuid() {
9436
+ const hex = "0123456789abcdef";
9437
+ const parts = [8, 4, 4, 4, 12];
9438
+ return parts.map((len) => {
9439
+ let s = "";
9440
+ for (let i = 0; i < len; i++) s += hex[Math.floor(Math.random() * 16)];
9441
+ return s;
9442
+ }).join("-");
9443
+ }
9444
+ function generateStripeId(prefix, index) {
9445
+ const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
9446
+ let id = prefix;
9447
+ let n = index + 42;
9448
+ for (let i = 0; i < 14; i++) {
9449
+ id += chars[n % chars.length];
9450
+ n = Math.floor(n / chars.length) + i + 7;
9451
+ }
9452
+ return id;
9453
+ }
9454
+ var DEFAULT_REACTIONS = {
9455
+ totalCount: 0,
9456
+ plusOne: 0,
9457
+ minusOne: 0,
9458
+ laugh: 0,
9459
+ hooray: 0,
9460
+ confused: 0,
9461
+ heart: 0,
9462
+ rocket: 0,
9463
+ eyes: 0
9464
+ };
9465
+ function resolveTemporalProperties(props, base) {
9466
+ const now = base ?? /* @__PURE__ */ new Date();
9467
+ if (props["stale"] === true) {
9468
+ const created2 = daysAgo(200 + Math.floor(Math.random() * 200), now);
9469
+ const updated2 = staleDate(91, 300, now);
9470
+ return { createdAt: created2, updatedAt: updated2 };
9471
+ }
9472
+ if (props["recentlyActive"] === true) {
9473
+ const created2 = daysAgo(30 + Math.floor(Math.random() * 100), now);
9474
+ const updated2 = recentDate(30, now);
9475
+ return { createdAt: created2, updatedAt: updated2 };
9476
+ }
9477
+ const created = daysAgo(14 + Math.floor(Math.random() * 60), now);
9478
+ const updated = recentDate(14, now);
9479
+ return { createdAt: created, updatedAt: updated };
9480
+ }
9481
+ function resolveLabels(props, availableLabels) {
9482
+ const labels = props["labels"];
9483
+ if (Array.isArray(labels)) return labels.map(String);
9484
+ return [];
9485
+ }
9486
+ function buildSeedFromBlueprint(blueprint, baseSeed) {
9487
+ const seed = structuredClone(baseSeed);
9488
+ const warnings = [];
9489
+ const now = /* @__PURE__ */ new Date();
9490
+ const existingLabels = /* @__PURE__ */ new Set();
9491
+ for (const label of seed["labels"] ?? []) {
9492
+ if (typeof label["name"] === "string") existingLabels.add(label["name"]);
9493
+ }
9494
+ for (const identity of blueprint.identities) {
9495
+ processIdentity(identity, seed, warnings);
9496
+ }
9497
+ const baseCollections = new Set(Object.keys(baseSeed));
9498
+ for (const spec of blueprint.collections) {
9499
+ if (!baseCollections.has(spec.name) && !seed[spec.name]) {
9500
+ warnings.push(`Blueprint references unknown collection "${spec.name}" \u2014 skipping`);
9501
+ warn(`Blueprint references unknown collection "${spec.name}" for ${blueprint.twin} twin \u2014 skipping`);
9502
+ continue;
9503
+ }
9504
+ processCollection(spec, seed, blueprint.twin, existingLabels, warnings, now);
9505
+ }
9506
+ return { seed, warnings };
9507
+ }
9508
+ function processIdentity(identity, seed, warnings) {
9509
+ const collection = identity.collection;
9510
+ if (!seed[collection]) {
9511
+ seed[collection] = [];
9512
+ }
9513
+ const existing = seed[collection].find((entity2) => {
9514
+ for (const [key, value] of Object.entries(identity.fields)) {
9515
+ if (entity2[key] !== value) return false;
9516
+ }
9517
+ return true;
9518
+ });
9519
+ if (existing) {
9520
+ debug(`Identity already exists in ${collection}: ${JSON.stringify(identity.fields)}`);
9521
+ return;
9522
+ }
9523
+ const nextId = getMaxId(seed[collection]) + 1;
9524
+ const now = (/* @__PURE__ */ new Date()).toISOString();
9525
+ const entity = {
9526
+ id: nextId,
9527
+ ...identity.fields,
9528
+ createdAt: now,
9529
+ updatedAt: now
9530
+ };
9531
+ fillDefaults(entity, collection);
9532
+ seed[collection].push(entity);
9533
+ debug(`Added identity to ${collection}: ${JSON.stringify(identity.fields)}`);
9534
+ }
9535
+ function processCollection(spec, seed, twinName, existingLabels, warnings, now) {
9536
+ const collection = spec.name;
9537
+ if (!seed[collection]) {
9538
+ seed[collection] = [];
9539
+ }
9540
+ const existingCount = seed[collection].length;
9541
+ if (existingCount > 0) {
9542
+ debug(`Clearing ${existingCount} existing ${collection} entities (blueprint replaces)`);
9543
+ seed[collection] = [];
9544
+ }
9545
+ const referencedLabels = /* @__PURE__ */ new Set();
9546
+ for (const group of spec.groups) {
9547
+ const labels = group.properties["labels"];
9548
+ if (Array.isArray(labels)) {
9549
+ for (const l of labels) {
9550
+ const name = String(l);
9551
+ if (!existingLabels.has(name)) referencedLabels.add(name);
9552
+ }
9553
+ }
9554
+ }
9555
+ if (referencedLabels.size > 0 && seed["labels"]) {
9556
+ const labelColors = ["d73a4a", "a2eeef", "0e8a16", "fbca04", "d876e3", "e4e669", "f9d0c4", "bfdadc"];
9557
+ let labelId = getMaxId(seed["labels"]) + 1;
9558
+ const repoId = findFirstRepoId(seed);
9559
+ for (const name of referencedLabels) {
9560
+ const color = labelColors[labelId % labelColors.length];
9561
+ seed["labels"].push({
9562
+ id: labelId,
9563
+ repoId: repoId ?? 1,
9564
+ nodeId: generateNodeId("LA_kwDOBlab", labelId),
9565
+ name,
9566
+ description: `Label: ${name}`,
9567
+ color,
9568
+ isDefault: false,
9569
+ createdAt: daysAgo(60, now),
9570
+ updatedAt: daysAgo(60, now)
9571
+ });
9572
+ existingLabels.add(name);
9573
+ labelId++;
9574
+ }
9575
+ }
9576
+ let entityIndex = 0;
9577
+ for (const group of spec.groups) {
9578
+ for (let i = 0; i < group.count; i++) {
9579
+ const entity = buildEntity(
9580
+ collection,
9581
+ twinName,
9582
+ group.properties,
9583
+ seed,
9584
+ entityIndex,
9585
+ spec.contentHint,
9586
+ now
9587
+ );
9588
+ seed[collection].push(entity);
9589
+ entityIndex++;
9590
+ }
9591
+ }
9592
+ debug(`Built ${entityIndex} entities for ${collection}`);
9593
+ }
9594
+ function buildEntity(collection, twinName, properties, seed, index, contentHint, now) {
9595
+ const nextId = getMaxId(seed[collection]) + 1;
9596
+ const temporal = resolveTemporalProperties(properties, now);
9597
+ if (twinName === "github") return buildGitHubEntity(collection, nextId, properties, seed, index, temporal, contentHint);
9598
+ if (twinName === "slack") return buildSlackEntity(collection, nextId, properties, seed, index, temporal, contentHint);
9599
+ if (twinName === "linear") return buildLinearEntity(collection, nextId, properties, seed, index, temporal, contentHint);
9600
+ if (twinName === "stripe") return buildStripeEntity(collection, nextId, properties, seed, index, temporal, contentHint);
9601
+ if (twinName === "jira") return buildJiraEntity(collection, nextId, properties, seed, index, temporal, contentHint);
9602
+ return {
9603
+ id: nextId,
9604
+ ...properties,
9605
+ createdAt: temporal.createdAt,
9606
+ updatedAt: temporal.updatedAt
9607
+ };
9608
+ }
9609
+ function buildGitHubEntity(collection, id, props, seed, index, temporal, contentHint) {
9610
+ const repoId = findFirstRepoId(seed) ?? 1;
9611
+ const owner = findRepoOwner(seed) ?? "acme";
9612
+ switch (collection) {
9613
+ case "issues": {
9614
+ const state = String(props["state"] ?? "open");
9615
+ const number = getMaxIssueNumber(seed) + 1;
9616
+ const labels = resolveLabels(props, []);
9617
+ const title = ISSUE_TITLES[index % ISSUE_TITLES.length];
9618
+ const body = ISSUE_BODIES[index % ISSUE_BODIES.length];
9619
+ const assigned = props["assigned"] === true;
9620
+ return {
9621
+ id,
9622
+ repoId,
9623
+ nodeId: generateNodeId("I_kwDOBiss", id),
9624
+ number,
9625
+ title: contentHint ? `${title} (${contentHint})` : title,
9626
+ body,
9627
+ state,
9628
+ stateReason: state === "closed" ? "completed" : null,
9629
+ locked: false,
9630
+ assignees: assigned ? [owner] : [],
9631
+ labels,
9632
+ milestone: null,
9633
+ authorLogin: owner,
9634
+ closedAt: state === "closed" ? temporal.updatedAt : null,
9635
+ closedBy: state === "closed" ? owner : null,
9636
+ htmlUrl: `https://github.com/${owner}/webapp/issues/${number}`,
9637
+ isPullRequest: false,
9638
+ reactions: { ...DEFAULT_REACTIONS },
9639
+ createdAt: temporal.createdAt,
9640
+ updatedAt: temporal.updatedAt
9641
+ };
9642
+ }
9643
+ case "pullRequests": {
9644
+ const state = String(props["state"] ?? "open");
9645
+ const number = getMaxPRNumber(seed) + 1;
9646
+ const title = PR_TITLES[index % PR_TITLES.length];
9647
+ const sha = generateSha();
9648
+ const baseSha = generateSha();
9649
+ return {
9650
+ id,
9651
+ repoId,
9652
+ nodeId: generateNodeId("PR_kwDOBpr", id),
9653
+ number,
9654
+ title,
9655
+ body: `This PR addresses ${title.toLowerCase()}.`,
9656
+ state,
9657
+ merged: props["merged"] === true || state === "closed",
9658
+ draft: props["draft"] === true,
9659
+ headRef: `feature/fix-${index}`,
9660
+ headSha: sha,
9661
+ baseRef: "main",
9662
+ baseSha,
9663
+ authorLogin: owner,
9664
+ labels: resolveLabels(props, []),
9665
+ assignees: [],
9666
+ requestedReviewers: [],
9667
+ milestone: null,
9668
+ additions: 10 + index * 5,
9669
+ deletions: 3 + index * 2,
9670
+ changedFiles: 1 + index % 5,
9671
+ commits: 1 + index % 3,
9672
+ comments: 0,
9673
+ reviewComments: 0,
9674
+ maintainerCanModify: true,
9675
+ mergeable: state === "open",
9676
+ mergedAt: props["merged"] === true ? temporal.updatedAt : null,
9677
+ mergedBy: props["merged"] === true ? owner : null,
9678
+ mergeCommitSha: props["merged"] === true ? generateSha() : null,
9679
+ closedAt: state === "closed" ? temporal.updatedAt : null,
9680
+ autoMerge: null,
9681
+ htmlUrl: `https://github.com/${owner}/webapp/pull/${number}`,
9682
+ diffUrl: `https://github.com/${owner}/webapp/pull/${number}.diff`,
9683
+ patchUrl: `https://github.com/${owner}/webapp/pull/${number}.patch`,
9684
+ createdAt: temporal.createdAt,
9685
+ updatedAt: temporal.updatedAt
9686
+ };
9687
+ }
9688
+ case "comments": {
9689
+ const issueNumber = resolveIssueNumberForComment(seed, index);
9690
+ return {
9691
+ id,
9692
+ repoId,
9693
+ nodeId: generateNodeId("IC_kwDOBcom", id),
9694
+ issueNumber,
9695
+ body: props["body"] ? String(props["body"]) : `Comment on issue #${issueNumber}`,
9696
+ authorLogin: owner,
9697
+ authorAssociation: "MEMBER",
9698
+ htmlUrl: `https://github.com/${owner}/webapp/issues/${issueNumber}#issuecomment-${id}`,
9699
+ reactions: { ...DEFAULT_REACTIONS },
9700
+ createdAt: temporal.createdAt,
9701
+ updatedAt: temporal.updatedAt
9702
+ };
9703
+ }
9704
+ case "labels": {
9705
+ const colors = ["d73a4a", "a2eeef", "0e8a16", "fbca04", "d876e3"];
9706
+ return {
9707
+ id,
9708
+ repoId,
9709
+ nodeId: generateNodeId("LA_kwDOBlab", id),
9710
+ name: props["name"] ? String(props["name"]) : `label-${id}`,
9711
+ description: props["description"] ? String(props["description"]) : null,
9712
+ color: colors[id % colors.length],
9713
+ isDefault: false,
9714
+ createdAt: temporal.createdAt,
9715
+ updatedAt: temporal.updatedAt
9716
+ };
9717
+ }
9718
+ default:
9719
+ return {
9720
+ id,
9721
+ repoId,
9722
+ ...props,
9723
+ createdAt: temporal.createdAt,
9724
+ updatedAt: temporal.updatedAt
9725
+ };
9726
+ }
9727
+ }
9728
+ function buildSlackEntity(collection, id, props, seed, index, temporal, contentHint) {
9729
+ const teamId = findSlackTeamId(seed) ?? "T0001TEAM";
9730
+ switch (collection) {
9731
+ case "channels": {
9732
+ const channelId = generateSlackId("C", index);
9733
+ const name = props["name"] ? String(props["name"]) : `channel-${index + 1}`;
9734
+ const userIds = (seed["users"] ?? []).map((u) => u["user_id"]).filter(Boolean);
9735
+ return {
9736
+ id,
9737
+ channel_id: channelId,
9738
+ name,
9739
+ is_channel: true,
9740
+ is_group: false,
9741
+ is_im: false,
9742
+ is_mpim: false,
9743
+ is_private: props["private"] === true,
9744
+ is_archived: false,
9745
+ is_general: name === "general",
9746
+ creator: userIds[0] ?? "U0001AAAA",
9747
+ topic: { value: "", creator: "", last_set: 0 },
9748
+ purpose: { value: CHANNEL_PURPOSES[index % CHANNEL_PURPOSES.length], creator: "", last_set: 0 },
9749
+ members: userIds,
9750
+ num_members: userIds.length,
9751
+ created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
9752
+ updated: Math.floor(new Date(temporal.updatedAt).getTime() / 1e3),
9753
+ createdAt: temporal.createdAt,
9754
+ updatedAt: temporal.updatedAt
9755
+ };
9756
+ }
9757
+ case "messages": {
9758
+ const channels = seed["channels"] ?? [];
9759
+ const targetChannel = channels.length > 0 ? channels[index % channels.length] : null;
9760
+ const channelId = targetChannel ? String(targetChannel["channel_id"] ?? "C0001AAAA") : "C0001AAAA";
9761
+ const channelMembers = targetChannel ? targetChannel["members"] ?? [] : [];
9762
+ const users = seed["users"] ?? [];
9763
+ let userId;
9764
+ if (channelMembers.length > 0) {
9765
+ userId = channelMembers[index % channelMembers.length];
9766
+ } else {
9767
+ userId = users.length > 0 ? String(users[index % users.length]["user_id"] ?? "U0001AAAA") : "U0001AAAA";
9768
+ }
9769
+ const baseTs = Math.floor(new Date(temporal.createdAt).getTime() / 1e3);
9770
+ const ts = generateSlackTs(baseTs, index);
9771
+ return {
9772
+ id,
9773
+ ts,
9774
+ channel_id: channelId,
9775
+ user_id: userId,
9776
+ text: props["text"] ? String(props["text"]) : MESSAGE_TEXTS[index % MESSAGE_TEXTS.length],
9777
+ type: "message",
9778
+ subtype: null,
9779
+ thread_ts: props["isReply"] === true ? generateSlackTs(baseTs, 0) : null,
9780
+ reply_count: 0,
9781
+ reply_users: [],
9782
+ reply_users_count: 0,
9783
+ latest_reply: null,
9784
+ createdAt: temporal.createdAt,
9785
+ updatedAt: temporal.updatedAt
9786
+ };
9787
+ }
9788
+ case "users": {
9789
+ const userId = generateSlackId("U", index);
9790
+ const name = props["name"] ? String(props["name"]) : `user${index + 1}`;
9791
+ return {
9792
+ id,
9793
+ user_id: userId,
9794
+ team_id: teamId,
9795
+ name,
9796
+ real_name: props["real_name"] ? String(props["real_name"]) : `User ${index + 1}`,
9797
+ display_name: name,
9798
+ email: `${name}@example.com`,
9799
+ is_admin: props["is_admin"] === true,
9800
+ is_owner: false,
9801
+ is_bot: props["is_bot"] === true,
9802
+ is_restricted: false,
9803
+ is_ultra_restricted: false,
9804
+ deleted: false,
9805
+ color: "4bbe2e",
9806
+ timezone: "America/Los_Angeles",
9807
+ tz_label: "Pacific Daylight Time",
9808
+ tz_offset: -25200,
9809
+ is_email_confirmed: true,
9810
+ who_can_share_contact_card: "EVERYONE",
9811
+ createdAt: temporal.createdAt,
9812
+ updatedAt: temporal.updatedAt
9813
+ };
9814
+ }
9815
+ default:
9816
+ return {
9817
+ id,
9818
+ ...props,
9819
+ createdAt: temporal.createdAt,
9820
+ updatedAt: temporal.updatedAt
9821
+ };
9822
+ }
9823
+ }
9824
+ function buildLinearEntity(collection, id, props, seed, index, temporal, contentHint) {
9825
+ switch (collection) {
9826
+ case "issues": {
9827
+ const teamId = findFirstId(seed, "teams") ?? 1;
9828
+ const team = (seed["teams"] ?? []).find((t) => t["id"] === teamId);
9829
+ const teamKey = team ? String(team["key"] ?? "ENG") : "ENG";
9830
+ const number = index + 1;
9831
+ const stateId = resolveLinearStateId(seed, props);
9832
+ const priority = typeof props["priority"] === "number" ? props["priority"] : 3;
9833
+ const priorityMap = {
9834
+ 0: "No priority",
9835
+ 1: "Urgent",
9836
+ 2: "High",
9837
+ 3: "Medium",
9838
+ 4: "Low"
9839
+ };
9840
+ return {
9841
+ id,
9842
+ linearId: generateUuid(),
9843
+ teamId,
9844
+ number,
9845
+ identifier: `${teamKey}-${number}`,
9846
+ title: ISSUE_TITLES[index % ISSUE_TITLES.length],
9847
+ description: null,
9848
+ descriptionData: null,
9849
+ priority,
9850
+ priorityLabel: priorityMap[priority] ?? "Medium",
9851
+ stateId,
9852
+ assigneeId: null,
9853
+ creatorId: null,
9854
+ projectId: null,
9855
+ cycleId: null,
9856
+ parentId: null,
9857
+ labelIds: [],
9858
+ estimate: null,
9859
+ dueDate: null,
9860
+ subIssueSortOrder: null,
9861
+ sortOrder: index * -100,
9862
+ snoozedUntilAt: null,
9863
+ startedAt: null,
9864
+ completedAt: null,
9865
+ canceledAt: null,
9866
+ archivedAt: null,
9867
+ trashed: false,
9868
+ createdAt: temporal.createdAt,
9869
+ updatedAt: temporal.updatedAt
9870
+ };
9871
+ }
9872
+ default:
9873
+ return {
9874
+ id,
9875
+ linearId: generateUuid(),
9876
+ ...props,
9877
+ createdAt: temporal.createdAt,
9878
+ updatedAt: temporal.updatedAt
9879
+ };
9880
+ }
9881
+ }
9882
+ function buildStripeEntity(collection, id, props, seed, index, temporal, contentHint) {
9883
+ switch (collection) {
9884
+ case "paymentIntents": {
9885
+ const piId = generateStripeId("pi_", index);
9886
+ const amount = typeof props["amount"] === "number" ? props["amount"] : 1e3 + index * 500;
9887
+ const status = String(props["status"] ?? "succeeded");
9888
+ return {
9889
+ id,
9890
+ paymentIntentId: piId,
9891
+ amount,
9892
+ currency: String(props["currency"] ?? "usd"),
9893
+ status,
9894
+ customerId: props["customerId"] ? String(props["customerId"]) : null,
9895
+ description: props["description"] ? String(props["description"]) : null,
9896
+ paymentMethodId: null,
9897
+ captureMethod: "automatic",
9898
+ confirmationMethod: "automatic",
9899
+ canceledAt: null,
9900
+ cancellationReason: null,
9901
+ latestChargeId: null,
9902
+ metadata: {},
9903
+ livemode: false,
9904
+ created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
9905
+ createdAt: temporal.createdAt,
9906
+ updatedAt: temporal.updatedAt
9907
+ };
9908
+ }
9909
+ case "customers": {
9910
+ const custId = generateStripeId("cus_", index);
9911
+ return {
9912
+ id,
9913
+ customerId: custId,
9914
+ name: props["name"] ? String(props["name"]) : `Customer ${index + 1}`,
9915
+ email: props["email"] ? String(props["email"]) : `customer${index + 1}@example.com`,
9916
+ phone: null,
9917
+ description: null,
9918
+ currency: null,
9919
+ defaultPaymentMethod: null,
9920
+ address: null,
9921
+ shipping: null,
9922
+ balance: 0,
9923
+ delinquent: false,
9924
+ metadata: {},
9925
+ livemode: false,
9926
+ created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
9927
+ createdAt: temporal.createdAt,
9928
+ updatedAt: temporal.updatedAt
9929
+ };
9930
+ }
9931
+ default:
9932
+ return {
9933
+ id,
9934
+ ...props,
9935
+ created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
9936
+ createdAt: temporal.createdAt,
9937
+ updatedAt: temporal.updatedAt
9938
+ };
9939
+ }
9940
+ }
9941
+ function buildJiraEntity(collection, id, props, seed, index, temporal, contentHint) {
9942
+ switch (collection) {
9943
+ case "issues": {
9944
+ const projectId = findFirstId(seed, "projects") ?? 1;
9945
+ const project = (seed["projects"] ?? []).find((p) => p["id"] === projectId);
9946
+ const projectKey = project ? String(project["key"] ?? "PROJ") : "PROJ";
9947
+ const number = index + 1;
9948
+ return {
9949
+ id,
9950
+ key: `${projectKey}-${number}`,
9951
+ projectId,
9952
+ issueTypeId: findFirstId(seed, "issueTypes") ?? 1,
9953
+ summary: ISSUE_TITLES[index % ISSUE_TITLES.length],
9954
+ description: null,
9955
+ statusId: findFirstId(seed, "statuses") ?? 1,
9956
+ priorityId: findFirstId(seed, "priorities") ?? 1,
9957
+ reporterAccountId: findFirstAccountId(seed),
9958
+ assigneeAccountId: props["assigned"] === true ? findFirstAccountId(seed) : null,
9959
+ parentKey: null,
9960
+ storyPoints: null,
9961
+ resolution: null,
9962
+ resolutionDate: null,
9963
+ labels: resolveLabels(props, []),
9964
+ components: [],
9965
+ fixVersions: [],
9966
+ createdAt: temporal.createdAt,
9967
+ updatedAt: temporal.updatedAt
9968
+ };
9969
+ }
9970
+ default:
9971
+ return {
9972
+ id,
9973
+ ...props,
9974
+ createdAt: temporal.createdAt,
9975
+ updatedAt: temporal.updatedAt
9976
+ };
9977
+ }
9978
+ }
9979
+ function findFirstRepoId(seed) {
9980
+ const repos = seed["repos"];
9981
+ return repos?.[0]?.["id"];
9982
+ }
9983
+ function findRepoOwner(seed) {
9984
+ const repos = seed["repos"];
9985
+ return repos?.[0]?.["owner"];
9986
+ }
9987
+ function findFirstId(seed, collection) {
9988
+ const entities = seed[collection];
9989
+ return entities?.[0]?.["id"];
9990
+ }
9991
+ function findFirstAccountId(seed) {
9992
+ const users = seed["users"];
9993
+ return String(users?.[0]?.["accountId"] ?? "account-1");
9994
+ }
9995
+ function findSlackTeamId(seed) {
9996
+ const workspaces = seed["workspaces"];
9997
+ if (workspaces?.[0]) return String(workspaces[0]["team_id"]);
9998
+ const users = seed["users"];
9999
+ if (users?.[0]) return String(users[0]["team_id"] ?? "T0001TEAM");
10000
+ return void 0;
10001
+ }
10002
+ function getMaxIssueNumber(seed) {
10003
+ let max = 0;
10004
+ for (const issue of seed["issues"] ?? []) {
10005
+ const n = issue["number"];
10006
+ if (typeof n === "number" && n > max) max = n;
10007
+ }
10008
+ return max;
10009
+ }
10010
+ function getMaxPRNumber(seed) {
10011
+ let max = 0;
10012
+ for (const pr of seed["pullRequests"] ?? []) {
10013
+ const n = pr["number"];
10014
+ if (typeof n === "number" && n > max) max = n;
10015
+ }
10016
+ return Math.max(max, getMaxIssueNumber(seed));
10017
+ }
10018
+ function resolveIssueNumberForComment(seed, index) {
10019
+ const issues = seed["issues"] ?? [];
10020
+ if (issues.length === 0) return 1;
10021
+ const issue = issues[index % issues.length];
10022
+ return issue["number"] ?? 1;
10023
+ }
10024
+ function resolveLinearStateId(seed, props) {
10025
+ const states = seed["workflowStates"] ?? [];
10026
+ if (states.length === 0) return 1;
10027
+ const state = props["state"];
10028
+ if (state === "completed" || state === "closed" || state === "done") {
10029
+ const found = states.find((s) => s["type"] === "completed");
10030
+ if (found) return found["id"];
10031
+ }
10032
+ if (state === "cancelled" || state === "canceled") {
10033
+ const found = states.find((s) => s["type"] === "cancelled");
10034
+ if (found) return found["id"];
10035
+ }
10036
+ if (state === "in_progress" || state === "started") {
10037
+ const found = states.find((s) => s["type"] === "started");
10038
+ if (found) return found["id"];
10039
+ }
10040
+ if (state === "backlog") {
10041
+ const found = states.find((s) => s["type"] === "backlog");
10042
+ if (found) return found["id"];
10043
+ }
10044
+ const unstarted = states.find((s) => s["type"] === "unstarted");
10045
+ return unstarted?.["id"] ?? states[0]["id"];
10046
+ }
10047
+ function fillDefaults(entity, collection) {
10048
+ switch (collection) {
10049
+ case "repos":
10050
+ entity["fullName"] = entity["fullName"] ?? `${entity["owner"] ?? "org"}/${entity["name"] ?? "repo"}`;
10051
+ entity["private"] = entity["private"] ?? false;
10052
+ entity["fork"] = entity["fork"] ?? false;
10053
+ entity["defaultBranch"] = entity["defaultBranch"] ?? "main";
10054
+ entity["archived"] = entity["archived"] ?? false;
10055
+ entity["disabled"] = entity["disabled"] ?? false;
10056
+ entity["visibility"] = entity["visibility"] ?? "public";
10057
+ entity["hasIssues"] = entity["hasIssues"] ?? true;
10058
+ entity["topics"] = entity["topics"] ?? [];
10059
+ entity["openIssuesCount"] = entity["openIssuesCount"] ?? 0;
10060
+ entity["stargazersCount"] = entity["stargazersCount"] ?? 0;
10061
+ entity["forksCount"] = entity["forksCount"] ?? 0;
10062
+ entity["watchersCount"] = entity["watchersCount"] ?? 0;
10063
+ break;
10064
+ case "users":
10065
+ entity["type"] = entity["type"] ?? "User";
10066
+ entity["siteAdmin"] = entity["siteAdmin"] ?? false;
10067
+ entity["publicRepos"] = entity["publicRepos"] ?? 0;
10068
+ entity["followers"] = entity["followers"] ?? 0;
10069
+ entity["following"] = entity["following"] ?? 0;
10070
+ break;
10071
+ }
10072
+ }
10073
+
9042
10074
  // src/runner/dynamic-seed-generator.ts
10075
+ function extractCountRequirements(setupText) {
10076
+ const requirements = [];
10077
+ const seen = /* @__PURE__ */ new Set();
10078
+ const NON_ENTITY = /* @__PURE__ */ new Set([
10079
+ "minutes",
10080
+ "minute",
10081
+ "hours",
10082
+ "hour",
10083
+ "days",
10084
+ "day",
10085
+ "weeks",
10086
+ "week",
10087
+ "months",
10088
+ "month",
10089
+ "years",
10090
+ "year",
10091
+ "seconds",
10092
+ "second",
10093
+ "ms",
10094
+ "am",
10095
+ "pm",
10096
+ "st",
10097
+ "nd",
10098
+ "rd",
10099
+ "th",
10100
+ "usd",
10101
+ "eur",
10102
+ "gbp",
10103
+ "percent",
10104
+ "kb",
10105
+ "mb",
10106
+ "gb",
10107
+ "tb"
10108
+ ]);
10109
+ const patterns = [
10110
+ /\b(\d+)\s+([\w\s]+?)(?:\s+(?:that|which|are|with|in|labeled|assigned|have|has)\b)/gi,
10111
+ /\b(\d+)\s+([\w\s]+?)(?:[.,;:)]|$)/gm
10112
+ ];
10113
+ for (const pattern of patterns) {
10114
+ for (const match of setupText.matchAll(pattern)) {
10115
+ const count = parseInt(match[1], 10);
10116
+ const subject = match[2].trim().toLowerCase();
10117
+ if (!subject || count <= 0 || count > 200) continue;
10118
+ const firstWord = subject.split(/\s+/)[0] ?? "";
10119
+ if (NON_ENTITY.has(firstWord)) continue;
10120
+ if (/^(?:of|and|or|the|those|these|them)\b/.test(subject)) continue;
10121
+ const key = `${count}:${subject}`;
10122
+ if (seen.has(key)) continue;
10123
+ seen.add(key);
10124
+ requirements.push(`- ${subject}: exactly ${count}`);
10125
+ }
10126
+ }
10127
+ return requirements;
10128
+ }
9043
10129
  var DynamicSeedError = class extends Error {
9044
10130
  twinName;
9045
10131
  validationErrors;
9046
10132
  constructor(twinName, validationErrors) {
9047
10133
  const details = validationErrors.length > 0 ? `:
9048
10134
  ${validationErrors.map((e) => ` - ${e}`).join("\n")}` : ".";
9049
- super(`Dynamic seed generation failed for twin "${twinName}"${details}`);
10135
+ const hint = "\n\nHint: Try `--static-seed` to skip dynamic generation and use pre-built seeds instead.";
10136
+ super(`Dynamic seed generation failed for twin "${twinName}"${details}${hint}`);
9050
10137
  this.name = "DynamicSeedError";
9051
10138
  this.twinName = twinName;
9052
10139
  this.validationErrors = validationErrors;
@@ -9280,6 +10367,14 @@ ${context.expectedBehavior}
9280
10367
  prompt += context.successCriteria.map((criterion) => `- ${criterion}`).join("\n");
9281
10368
  prompt += "\n\n";
9282
10369
  }
10370
+ const countReqs = extractCountRequirements(setupDescription);
10371
+ if (countReqs.length > 0) {
10372
+ prompt += `## MANDATORY Count Requirements
10373
+ The final seed state MUST have these exact counts. Do NOT deviate:
10374
+ `;
10375
+ prompt += countReqs.join("\n");
10376
+ prompt += "\n\n";
10377
+ }
9283
10378
  prompt += `## Setup Description
9284
10379
  This is a multi-service scenario. Generate ONLY the ${twinName} seed data. Ignore setup details that belong to other services. Faithfully reproduce EVERY detail relevant to ${twinName}. Specific names, messages, amounts, and entities mentioned MUST exist in the generated data.
9285
10380
 
@@ -9682,6 +10777,87 @@ function parseSeedPatchResponse(text, twinName) {
9682
10777
  });
9683
10778
  return null;
9684
10779
  }
10780
+ async function tryBlueprintPath(twinName, baseSeedData, setupDescription, availableCollections, config, intent) {
10781
+ try {
10782
+ const blueprintConfig = {
10783
+ apiKey: config.apiKey,
10784
+ model: config.model,
10785
+ baseUrl: config.baseUrl,
10786
+ providerMode: config.providerMode
10787
+ };
10788
+ const blueprint = await extractBlueprint(
10789
+ setupDescription,
10790
+ twinName,
10791
+ availableCollections,
10792
+ blueprintConfig
10793
+ );
10794
+ if (!blueprint) {
10795
+ debug("Blueprint extraction returned null");
10796
+ return null;
10797
+ }
10798
+ if (blueprint.collections.length === 0 && blueprint.identities.length === 0) {
10799
+ debug("Blueprint extraction returned empty blueprint");
10800
+ return null;
10801
+ }
10802
+ debug("Blueprint extracted", {
10803
+ identities: String(blueprint.identities.length),
10804
+ collections: blueprint.collections.map((c) => `${c.name}:${c.totalCount}`).join(", ")
10805
+ });
10806
+ const { seed: builtSeed, warnings } = buildSeedFromBlueprint(blueprint, baseSeedData);
10807
+ if (warnings.length > 0) {
10808
+ debug("Blueprint builder warnings", { warnings: warnings.join("; ") });
10809
+ }
10810
+ let finalSeed = normalizeSeedData(builtSeed, twinName);
10811
+ finalSeed = autoFillMissingFKs(finalSeed, twinName);
10812
+ const relValidation = validateSeedRelationships(finalSeed, twinName);
10813
+ if (!relValidation.valid) {
10814
+ warn("Blueprint seed failed relationship validation", {
10815
+ errors: relValidation.errors.slice(0, 5).join("; ")
10816
+ });
10817
+ return null;
10818
+ }
10819
+ if (intent) {
10820
+ const coverage = validateSeedCoverage(intent, finalSeed);
10821
+ if (!coverage.valid) {
10822
+ warn("Blueprint seed failed coverage validation", {
10823
+ errors: coverage.issues.map((i) => i.message).join("; ")
10824
+ });
10825
+ return null;
10826
+ }
10827
+ }
10828
+ const flatForVerify = {};
10829
+ flatForVerify[twinName] = finalSeed;
10830
+ const countMismatches = verifySeedCounts(setupDescription, flatForVerify);
10831
+ if (countMismatches.length > 0) {
10832
+ debug("Blueprint seed has count mismatches (acceptable)", {
10833
+ mismatches: countMismatches.map((m) => `${m.subject}: ${m.expected} vs ${m.actual}`).join("; ")
10834
+ });
10835
+ }
10836
+ const syntheticPatch = {
10837
+ add: {}
10838
+ };
10839
+ for (const [collection, entities] of Object.entries(finalSeed)) {
10840
+ const baseCount = baseSeedData[collection]?.length ?? 0;
10841
+ if (entities.length > baseCount) {
10842
+ syntheticPatch.add[collection] = entities.slice(baseCount).map((e) => {
10843
+ const { id: _id, createdAt: _ca, updatedAt: _ua, ...rest } = e;
10844
+ return rest;
10845
+ });
10846
+ }
10847
+ }
10848
+ return {
10849
+ seed: finalSeed,
10850
+ patch: syntheticPatch,
10851
+ fromCache: false,
10852
+ source: "llm"
10853
+ // Blueprint still uses LLM for extraction
10854
+ };
10855
+ } catch (err) {
10856
+ const msg = err instanceof Error ? err.message : String(err);
10857
+ warn(`Blueprint path failed: ${msg}`);
10858
+ return null;
10859
+ }
10860
+ }
9685
10861
  async function generateDynamicSeed(twinName, baseSeedName, baseSeedData, setupDescription, config, intent, context) {
9686
10862
  const cacheScope = {
9687
10863
  baseSeedData,
@@ -9703,6 +10879,28 @@ async function generateDynamicSeed(twinName, baseSeedName, baseSeedData, setupDe
9703
10879
  "No API key configured for seed generation. Set ARCHAL_TOKEN or configure a provider API key."
9704
10880
  ]);
9705
10881
  }
10882
+ progress(`Generating dynamic seed for ${twinName}...`);
10883
+ const availableCollections = Object.keys(baseSeedData);
10884
+ const blueprintResult = await tryBlueprintPath(
10885
+ twinName,
10886
+ baseSeedData,
10887
+ setupDescription,
10888
+ availableCollections,
10889
+ config,
10890
+ intent
10891
+ );
10892
+ if (blueprintResult) {
10893
+ info("Dynamic seed generated via blueprint", { twin: twinName });
10894
+ if (!config.noCache) {
10895
+ const cacheContext = buildSeedCacheContext(twinName, intent, context);
10896
+ cacheSeed(twinName, baseSeedName, setupDescription, blueprintResult.seed, blueprintResult.patch, {
10897
+ baseSeedData,
10898
+ cacheContext
10899
+ });
10900
+ }
10901
+ return blueprintResult;
10902
+ }
10903
+ debug("Blueprint path failed or produced invalid seed, falling back to full LLM generation");
9706
10904
  const userPrompt = buildSeedGenerationPrompt(
9707
10905
  twinName,
9708
10906
  baseSeedData,
@@ -9710,7 +10908,6 @@ async function generateDynamicSeed(twinName, baseSeedName, baseSeedData, setupDe
9710
10908
  intent,
9711
10909
  context
9712
10910
  );
9713
- progress(`Generating dynamic seed for ${twinName}...`);
9714
10911
  let patch = null;
9715
10912
  let mergedSeed = null;
9716
10913
  let lastErrors = [];
@@ -9741,7 +10938,7 @@ Fix these issues:
9741
10938
  validationAttempt: String(validationAttempts + 1)
9742
10939
  });
9743
10940
  const provider = detectProvider(config.model);
9744
- const apiKey = resolveProviderApiKey(config.apiKey, provider);
10941
+ const apiKey = effectiveMode === "archal" ? "" : resolveProviderApiKey(config.apiKey, provider);
9745
10942
  const responseText = await callLlm({
9746
10943
  provider,
9747
10944
  model: config.model,
@@ -9750,7 +10947,7 @@ Fix these issues:
9750
10947
  userPrompt: promptWithFeedback,
9751
10948
  maxTokens: 16384,
9752
10949
  baseUrl: config.baseUrl,
9753
- providerMode: config.providerMode,
10950
+ providerMode: effectiveMode,
9754
10951
  intent: "seed-generate",
9755
10952
  responseFormat: "json"
9756
10953
  });
@@ -9860,6 +11057,22 @@ Fix these issues:
9860
11057
  continue;
9861
11058
  }
9862
11059
  }
11060
+ if (mergedSeed && setupDescription && validationAttempts < MAX_ATTEMPTS - 1) {
11061
+ const flatForVerify = {};
11062
+ flatForVerify[twinName] = mergedSeed;
11063
+ const countMismatches = verifySeedCounts(setupDescription, flatForVerify);
11064
+ if (countMismatches.length > 0) {
11065
+ const countErrors = countMismatches.map(
11066
+ (m) => `Count mismatch: "${m.subject}" should be ${m.expected} but got ${m.actual}`
11067
+ );
11068
+ warn(`Seed count mismatch (attempt ${attempt + 1})`, {
11069
+ errors: countErrors.join("; ")
11070
+ });
11071
+ lastErrors = countErrors;
11072
+ validationAttempts++;
11073
+ continue;
11074
+ }
11075
+ }
9863
11076
  break;
9864
11077
  } catch (err) {
9865
11078
  const message = err instanceof Error ? err.message : String(err);
@@ -10835,11 +12048,21 @@ function parseSqlSeed(sql) {
10835
12048
  function loadSeedStateFromPath(seedRoot, seedName) {
10836
12049
  const jsonPath = resolve4(seedRoot, `${seedName}.json`);
10837
12050
  if (existsSync10(jsonPath)) {
10838
- return JSON.parse(readFileSync12(jsonPath, "utf-8"));
12051
+ try {
12052
+ return JSON.parse(readFileSync12(jsonPath, "utf-8"));
12053
+ } catch (err) {
12054
+ const detail = err instanceof Error ? err.message : String(err);
12055
+ throw new Error(`Failed to parse seed file ${jsonPath}: ${detail}`);
12056
+ }
10839
12057
  }
10840
12058
  const sqlPath = resolve4(seedRoot, `${seedName}.sql`);
10841
12059
  if (existsSync10(sqlPath)) {
10842
- return parseSqlSeed(readFileSync12(sqlPath, "utf-8"));
12060
+ try {
12061
+ return parseSqlSeed(readFileSync12(sqlPath, "utf-8"));
12062
+ } catch (err) {
12063
+ const detail = err instanceof Error ? err.message : String(err);
12064
+ throw new Error(`Failed to parse seed file ${sqlPath}: ${detail}`);
12065
+ }
10843
12066
  }
10844
12067
  return null;
10845
12068
  }
@@ -11075,7 +12298,9 @@ ${baseTaskMessage}` : baseTaskMessage;
11075
12298
  };
11076
12299
  }
11077
12300
  if (trace.length === 0) {
11078
- warn(`Agent made no tool calls on run ${runIndex + 1}. The agent may have failed to act \u2014 check agent logs and task prompt.`);
12301
+ warn(
12302
+ `Agent made no tool calls on run ${runIndex + 1}. This usually means the model is too weak for this scenario. Try a more capable model (e.g. --engine-model claude-sonnet-4-6 or --engine-model gemini-2.5-pro). If using a custom agent, check that it correctly processes tool schemas and calls tools.`
12303
+ );
11079
12304
  }
11080
12305
  progress(`Evaluating run ${runIndex + 1}...`);
11081
12306
  const evaluationResult = await evaluateRun(
@@ -11412,8 +12637,14 @@ Pass --allow-ambiguous-seed to opt into best-effort generation.`;
11412
12637
  for (const sel of seedSelections) {
11413
12638
  const mismatches = verifySeedCounts(scenario.setup, sel.seedData);
11414
12639
  if (mismatches.length === 0) continue;
12640
+ const significantMismatches = mismatches.filter((m) => {
12641
+ const delta = Math.abs(m.expected - m.actual);
12642
+ const ratio = m.expected > 0 ? delta / m.expected : delta;
12643
+ return delta > 5 || ratio > 0.5;
12644
+ });
12645
+ if (significantMismatches.length === 0) continue;
11415
12646
  warn(
11416
- `Seed count mismatch for ${sel.twinName}: ${mismatches.map((m) => `${m.subject}: expected ${m.expected}, got ${m.actual}`).join("; ")}`
12647
+ `Seed count mismatch for ${sel.twinName}: ${significantMismatches.map((m) => `${m.subject}: expected ${m.expected}, got ${m.actual}`).join("; ")}`
11417
12648
  );
11418
12649
  }
11419
12650
  const scenarioDir = dirname2(resolve4(options.scenarioPath));
@@ -11605,7 +12836,7 @@ Pass --allow-ambiguous-seed to opt into best-effort generation.`;
11605
12836
  printHeader(scenario.title, seedSelections);
11606
12837
  const evaluatorProvider = detectProvider(model);
11607
12838
  const configProvider = detectProvider(config.model);
11608
- const evaluatorApiKey = options.model && evaluatorProvider !== configProvider ? resolveProviderApiKey("", evaluatorProvider) : resolveProviderApiKey(config.apiKey, evaluatorProvider);
12839
+ const evaluatorApiKey = config.evaluatorProvider === "archal" ? "" : options.model && evaluatorProvider !== configProvider ? resolveProviderApiKey("", evaluatorProvider) : resolveProviderApiKey(config.apiKey, evaluatorProvider);
11609
12840
  const evaluatorConfig = {
11610
12841
  apiKey: evaluatorApiKey,
11611
12842
  model,
@@ -12132,7 +13363,10 @@ function createRunCommand() {
12132
13363
  ).addOption(new Option("--openclaw-url <url>", "Deprecated alias for --engine-endpoint").hideHelp()).addOption(new Option("--openclaw-token <token>", "Deprecated alias for --engine-token").hideHelp()).addOption(new Option("--openclaw-agent <id>", "Deprecated alias for --engine-model").hideHelp()).addOption(new Option("--openclaw-twin-urls <path>", "Deprecated alias for --engine-twin-urls").hideHelp()).addOption(new Option("--openclaw-timeout <seconds>", "Deprecated alias for --engine-timeout").hideHelp()).option("--api-base-urls <path>", "Path to JSON mapping service names to clone API base URLs for raw API code routing").option("--api-proxy-url <url>", "Proxy URL for raw API code routing metadata").option("--preflight-only", "Run environment/config preflight checks only and exit").option("--seed-cache", "Enable seed cache for dynamic generation (off by default)").option("--static-seed", "Use seed files as-is without LLM mutation (uses --seed name or auto-selected per twin)").option("--no-failure-analysis", "Skip LLM failure analysis on imperfect scores").option(
12133
13364
  "--allow-ambiguous-seed",
12134
13365
  "Allow dynamic seed generation when setup is underspecified"
12135
- ).option("--tag <tag>", "Only run if scenario has this tag (exit 0 if not)").option("-q, --quiet", "Suppress non-error output").option("-v, --verbose", "Enable debug logging").action(async (scenarioArg, opts) => {
13366
+ ).option("--tag <tag>", "Only run if scenario has this tag (exit 0 if not)").option("-q, --quiet", "Suppress non-error output").option("-v, --verbose", "Enable debug logging").action(async (scenarioArg, opts, command) => {
13367
+ const parentOpts = command.parent?.opts() ?? {};
13368
+ if (parentOpts.quiet) opts.quiet = true;
13369
+ if (parentOpts.verbose) opts.verbose = true;
12136
13370
  if (opts.quiet) {
12137
13371
  configureLogger({ quiet: true });
12138
13372
  }
@@ -12405,9 +13639,16 @@ function createRunCommand() {
12405
13639
  }
12406
13640
  if (opts.apiKey?.trim()) {
12407
13641
  warnIfKeyLooksInvalid(opts.apiKey.trim(), "--api-key");
12408
- process.env["ARCHAL_ENGINE_API_KEY"] = opts.apiKey.trim();
13642
+ const key = opts.apiKey.trim();
13643
+ process.env["ARCHAL_ENGINE_API_KEY"] = key;
13644
+ if (key.startsWith("AIza")) {
13645
+ process.env["GEMINI_API_KEY"] = process.env["GEMINI_API_KEY"] || key;
13646
+ } else if (key.startsWith("sk-ant-")) {
13647
+ process.env["ANTHROPIC_API_KEY"] = process.env["ANTHROPIC_API_KEY"] || key;
13648
+ } else if (key.startsWith("sk-")) {
13649
+ process.env["OPENAI_API_KEY"] = process.env["OPENAI_API_KEY"] || key;
13650
+ }
12409
13651
  if (!opts.engineModel && !process.env["ARCHAL_ENGINE_MODEL"] && !opts.model?.trim()) {
12410
- const key = opts.apiKey.trim();
12411
13652
  if (key.startsWith("AIza")) {
12412
13653
  opts.engineModel = "gemini-2.0-flash";
12413
13654
  } else if (key.startsWith("sk-ant-")) {
@@ -12578,13 +13819,13 @@ function createRunCommand() {
12578
13819
  let statusReadySinceMs = null;
12579
13820
  const isRetryablePollFailure = (result) => result.offline || typeof result.status === "number" && result.status >= 500;
12580
13821
  const sleepForPollInterval = async () => new Promise((resolve12) => setTimeout(resolve12, SESSION_POLL_INTERVAL_MS));
12581
- process.stderr.write("Starting cloud session...\n");
13822
+ if (!opts.quiet) process.stderr.write("Starting cloud session...\n");
12582
13823
  let pollCount = 0;
12583
13824
  while (Date.now() < readyDeadline) {
12584
13825
  pollCount++;
12585
13826
  if (pollCount % 4 === 0) {
12586
13827
  const elapsedSec = Math.round((Date.now() - (readyDeadline - SESSION_READY_TIMEOUT_MS)) / 1e3);
12587
- process.stderr.write(` Still waiting for session to be ready (${elapsedSec}s)...
13828
+ if (!opts.quiet) process.stderr.write(` Still waiting for session to be ready (${elapsedSec}s)...
12588
13829
  `);
12589
13830
  }
12590
13831
  const freshCreds = getCredentials();
@@ -12655,7 +13896,7 @@ function createRunCommand() {
12655
13896
  }
12656
13897
  if (sessionReady) {
12657
13898
  const warmupSec = Math.round((Date.now() - (readyDeadline - SESSION_READY_TIMEOUT_MS)) / 1e3);
12658
- process.stderr.write(`Cloud session ready (${warmupSec}s).
13899
+ if (!opts.quiet) process.stderr.write(`Cloud session ready (${warmupSec}s).
12659
13900
  `);
12660
13901
  }
12661
13902
  if (!sessionReady && !runFailureMessage) {
@@ -13018,12 +14259,33 @@ function buildEvidenceArtifacts(report) {
13018
14259
  runIndex: run.runIndex,
13019
14260
  steps: buildAgentTraceSteps(run)
13020
14261
  })).filter((run) => run.steps.length > 0);
14262
+ const evaluations = reportRuns.flatMap(
14263
+ (run) => (run.evaluations ?? []).map((ev) => ({
14264
+ runIndex: run.runIndex,
14265
+ criterionId: ev.criterionId,
14266
+ status: ev.status,
14267
+ result: ev.status,
14268
+ confidence: ev.confidence,
14269
+ explanation: ev.explanation
14270
+ }))
14271
+ );
14272
+ const latestEvaluationByCriterion = /* @__PURE__ */ new Map();
14273
+ for (const evaluation of evaluations) {
14274
+ const current = latestEvaluationByCriterion.get(evaluation.criterionId);
14275
+ if (!current || evaluation.runIndex >= current.runIndex) {
14276
+ latestEvaluationByCriterion.set(evaluation.criterionId, evaluation);
14277
+ }
14278
+ }
13021
14279
  const criteria = Object.entries(report.criterionDescriptions ?? {}).map(
13022
- ([id, description]) => ({
13023
- id,
13024
- label: description,
13025
- kind: report.criterionTypes?.[id] ?? null
13026
- })
14280
+ ([id, description]) => {
14281
+ const latest = latestEvaluationByCriterion.get(id);
14282
+ return {
14283
+ id,
14284
+ label: description,
14285
+ kind: report.criterionTypes?.[id] ?? null,
14286
+ result: latest?.result ?? null
14287
+ };
14288
+ }
13027
14289
  );
13028
14290
  const runs = reportRuns.map((run) => ({
13029
14291
  runIndex: run.runIndex,
@@ -13033,6 +14295,7 @@ function buildEvidenceArtifacts(report) {
13033
14295
  evaluations: (run.evaluations ?? []).map((ev) => ({
13034
14296
  criterionId: ev.criterionId,
13035
14297
  status: ev.status,
14298
+ result: ev.status,
13036
14299
  confidence: ev.confidence,
13037
14300
  explanation: ev.explanation
13038
14301
  }))
@@ -13041,6 +14304,7 @@ function buildEvidenceArtifacts(report) {
13041
14304
  satisfaction: report.satisfactionScore,
13042
14305
  scores: reportRuns.map((r) => r.overallScore),
13043
14306
  criteria,
14307
+ evaluations,
13044
14308
  runs,
13045
14309
  traceEntries,
13046
14310
  thinkingTraceEntries,
@@ -15118,9 +16382,10 @@ function findBundledScenarios() {
15118
16382
  return results;
15119
16383
  }
15120
16384
  function detectProviderName(model) {
15121
- if (model.startsWith("gemini-")) return "Gemini";
15122
- if (model.startsWith("claude-")) return "Anthropic";
15123
- if (model.startsWith("gpt-") || model.startsWith("o1-") || model.startsWith("o3-") || model.startsWith("o4-")) return "OpenAI";
16385
+ const normalized = model.toLowerCase();
16386
+ if (normalized.startsWith("gemini-")) return "Gemini";
16387
+ if (normalized.startsWith("claude-") || normalized.startsWith("sonnet-") || normalized.startsWith("haiku-") || normalized.startsWith("opus-")) return "Anthropic";
16388
+ if (normalized.startsWith("gpt-") || normalized.startsWith("o1-") || normalized.startsWith("o3-") || normalized.startsWith("o4-")) return "OpenAI";
15124
16389
  return "OpenAI-compatible";
15125
16390
  }
15126
16391
  function resolveEngineApiKey(explicitKey) {
@@ -15189,6 +16454,13 @@ Set one via:
15189
16454
  process.exit(1);
15190
16455
  }
15191
16456
  process.env["ARCHAL_ENGINE_API_KEY"] = engineApiKey;
16457
+ if (engineApiKey.startsWith("AIza")) {
16458
+ process.env["GEMINI_API_KEY"] = process.env["GEMINI_API_KEY"] || engineApiKey;
16459
+ } else if (engineApiKey.startsWith("sk-ant-")) {
16460
+ process.env["ANTHROPIC_API_KEY"] = process.env["ANTHROPIC_API_KEY"] || engineApiKey;
16461
+ } else if (engineApiKey.startsWith("sk-")) {
16462
+ process.env["OPENAI_API_KEY"] = process.env["OPENAI_API_KEY"] || engineApiKey;
16463
+ }
15192
16464
  const runs = parseInt(opts.runs, 10);
15193
16465
  if (Number.isNaN(runs) || runs <= 0) {
15194
16466
  process.stderr.write("Error: --runs must be a positive integer\n");