@archal/cli 0.7.7 → 0.7.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1326 -54
- package/harnesses/_lib/model-configs.mjs +26 -19
- package/harnesses/_lib/providers.mjs +24 -8
- package/harnesses/hardened/agent.mjs +25 -105
- package/harnesses/naive/agent.mjs +2 -2
- package/harnesses/react/agent.mjs +2 -2
- package/harnesses/zero-shot/agent.mjs +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -233,6 +233,7 @@ function parseCriterionLine(line, index) {
|
|
|
233
233
|
} else {
|
|
234
234
|
type = inferCriterionType(description);
|
|
235
235
|
}
|
|
236
|
+
if (!description) return null;
|
|
236
237
|
return {
|
|
237
238
|
id: `criterion-${index + 1}`,
|
|
238
239
|
description,
|
|
@@ -333,7 +334,11 @@ ${expectedBehavior}`.toLowerCase();
|
|
|
333
334
|
github: ["github", "repository", "pull request", "create_issue", "create_pull_request", "merge_pull_request"],
|
|
334
335
|
slack: ["slack", "slack channel", "send_message", "slack message", "direct message"],
|
|
335
336
|
linear: ["linear", "linear ticket", "linear project", "linear cycle"],
|
|
336
|
-
jira: ["jira", "jira sprint", "jira epic", "jira board"]
|
|
337
|
+
jira: ["jira", "jira sprint", "jira epic", "jira board"],
|
|
338
|
+
stripe: ["stripe", "payment", "refund", "subscription", "invoice", "charge"],
|
|
339
|
+
supabase: ["supabase", "database", "sql query", "database table"],
|
|
340
|
+
"google-workspace": ["google workspace", "gmail", "google calendar", "google drive", "google docs"],
|
|
341
|
+
browser: ["browser", "web page", "navigate to", "click on", "web content"]
|
|
337
342
|
};
|
|
338
343
|
for (const [twin, keywords] of Object.entries(twinKeywords)) {
|
|
339
344
|
if (keywords.some((kw) => combined.includes(kw))) {
|
|
@@ -425,7 +430,9 @@ function validateScenario(scenario) {
|
|
|
425
430
|
}
|
|
426
431
|
}
|
|
427
432
|
if (scenario.config.twins.length === 0) {
|
|
428
|
-
errors.push(
|
|
433
|
+
errors.push(
|
|
434
|
+
'Scenario does not reference any known twins. Add a "## Config" section with "twins: github" (or slack, linear, jira, stripe, supabase, google-workspace, browser). Alternatively, mention the service name in ## Setup or ## Expected Behavior.'
|
|
435
|
+
);
|
|
429
436
|
}
|
|
430
437
|
if (scenario.config.timeout <= 0) {
|
|
431
438
|
errors.push("Timeout must be a positive number");
|
|
@@ -2982,10 +2989,11 @@ var LlmApiError = class extends Error {
|
|
|
2982
2989
|
};
|
|
2983
2990
|
var RETRYABLE_STATUS_CODES2 = /* @__PURE__ */ new Set([429, 500, 502, 503, 529]);
|
|
2984
2991
|
function detectProvider(model) {
|
|
2985
|
-
|
|
2986
|
-
if (
|
|
2987
|
-
if (
|
|
2988
|
-
if (
|
|
2992
|
+
const normalized = model.toLowerCase();
|
|
2993
|
+
if (normalized.startsWith("gemini-")) return "gemini";
|
|
2994
|
+
if (normalized.startsWith("claude-") || normalized.startsWith("sonnet-") || normalized.startsWith("haiku-") || normalized.startsWith("opus-")) return "anthropic";
|
|
2995
|
+
if (normalized.startsWith("gpt-") || normalized.startsWith("o1-") || normalized.startsWith("o2-") || normalized.startsWith("o3-") || normalized.startsWith("o4-")) return "openai";
|
|
2996
|
+
if (normalized.startsWith("llama") || normalized.startsWith("mixtral") || normalized.startsWith("mistral") || normalized.startsWith("deepseek") || normalized.startsWith("qwen") || normalized.startsWith("codestral") || normalized.startsWith("command")) return "openai-compatible";
|
|
2989
2997
|
return "openai-compatible";
|
|
2990
2998
|
}
|
|
2991
2999
|
var PROVIDER_ENV_VARS = {
|
|
@@ -3011,8 +3019,18 @@ function validateKeyForProvider(key, provider) {
|
|
|
3011
3019
|
return void 0;
|
|
3012
3020
|
}
|
|
3013
3021
|
function resolveProviderApiKey(explicitKey, provider) {
|
|
3014
|
-
|
|
3015
|
-
|
|
3022
|
+
const normalizedExplicit = explicitKey.trim();
|
|
3023
|
+
const providerEnvKey = process.env[PROVIDER_ENV_VARS[provider]]?.trim() ?? "";
|
|
3024
|
+
if (!normalizedExplicit) return providerEnvKey;
|
|
3025
|
+
const mismatch = validateKeyForProvider(normalizedExplicit, provider);
|
|
3026
|
+
if (mismatch && providerEnvKey) {
|
|
3027
|
+
debug("Configured API key appears mismatched for provider; falling back to provider env key", {
|
|
3028
|
+
provider,
|
|
3029
|
+
providerEnvVar: PROVIDER_ENV_VARS[provider]
|
|
3030
|
+
});
|
|
3031
|
+
return providerEnvKey;
|
|
3032
|
+
}
|
|
3033
|
+
return normalizedExplicit;
|
|
3016
3034
|
}
|
|
3017
3035
|
var REQUEST_TIMEOUT_MS3 = 6e4;
|
|
3018
3036
|
var MAX_RETRIES2 = 3;
|
|
@@ -3061,7 +3079,7 @@ async function callLlmViaArchal(options) {
|
|
|
3061
3079
|
debug("Archal backend response", { model: actualModel, remaining: String(result.data.remaining ?? "unknown") });
|
|
3062
3080
|
const isSeedGen = options.intent === "seed-generate";
|
|
3063
3081
|
if (!modelMismatchWarned && !isSeedGen && options.model && actualModel && !actualModel.includes(options.model) && !options.model.includes(actualModel)) {
|
|
3064
|
-
|
|
3082
|
+
debug(`Archal backend used "${actualModel}" (requested "${options.model}"). To use a specific model, set provider to "direct" with your own API key.`);
|
|
3065
3083
|
modelMismatchWarned = true;
|
|
3066
3084
|
}
|
|
3067
3085
|
return result.data.text;
|
|
@@ -4182,8 +4200,35 @@ function filterByPredicate(items, predicate) {
|
|
|
4182
4200
|
if (knownMatches.length > 0) {
|
|
4183
4201
|
return { items: knownMatches, recognized: true };
|
|
4184
4202
|
}
|
|
4203
|
+
const ACTION_VERBS = /* @__PURE__ */ new Set([
|
|
4204
|
+
"listed",
|
|
4205
|
+
"fetched",
|
|
4206
|
+
"retrieved",
|
|
4207
|
+
"found",
|
|
4208
|
+
"searched",
|
|
4209
|
+
"queried",
|
|
4210
|
+
"posted",
|
|
4211
|
+
"sent",
|
|
4212
|
+
"received",
|
|
4213
|
+
"notified",
|
|
4214
|
+
"alerted",
|
|
4215
|
+
"reviewed",
|
|
4216
|
+
"analyzed",
|
|
4217
|
+
"inspected",
|
|
4218
|
+
"checked",
|
|
4219
|
+
"verified",
|
|
4220
|
+
"triaged",
|
|
4221
|
+
"escalated",
|
|
4222
|
+
"assigned",
|
|
4223
|
+
"tagged",
|
|
4224
|
+
"labeled",
|
|
4225
|
+
"updated",
|
|
4226
|
+
"edited",
|
|
4227
|
+
"patched",
|
|
4228
|
+
"migrated"
|
|
4229
|
+
]);
|
|
4185
4230
|
const isSingleWord = !lowerPredicate.includes(" ");
|
|
4186
|
-
if (isSingleWord) {
|
|
4231
|
+
if (isSingleWord && !ACTION_VERBS.has(lowerPredicate)) {
|
|
4187
4232
|
const hasKnownField = items.some((item) => {
|
|
4188
4233
|
if (typeof item !== "object" || item === null) return false;
|
|
4189
4234
|
const obj = item;
|
|
@@ -5455,24 +5500,46 @@ ${JSON.stringify(context.stateDiff, null, 2)}
|
|
|
5455
5500
|
## Agent Trace Evidence
|
|
5456
5501
|
${traceEvidence}`;
|
|
5457
5502
|
}
|
|
5503
|
+
function estimateTokens(value) {
|
|
5504
|
+
const json = JSON.stringify(value);
|
|
5505
|
+
return Math.ceil(json.length / 4);
|
|
5506
|
+
}
|
|
5507
|
+
var MAX_STATE_TOKENS = 4e4;
|
|
5458
5508
|
function summarizeState(state) {
|
|
5459
5509
|
const flat = flattenTwinState(state);
|
|
5460
5510
|
const summary = {};
|
|
5461
5511
|
for (const [key, value] of Object.entries(flat)) {
|
|
5462
5512
|
if (Array.isArray(value)) {
|
|
5463
|
-
if (value.length <=
|
|
5513
|
+
if (value.length <= 50) {
|
|
5464
5514
|
summary[key] = value;
|
|
5465
5515
|
} else {
|
|
5466
5516
|
summary[key] = {
|
|
5467
5517
|
_count: value.length,
|
|
5468
|
-
|
|
5469
|
-
|
|
5518
|
+
_first10: value.slice(0, 10),
|
|
5519
|
+
_last10: value.slice(-10)
|
|
5470
5520
|
};
|
|
5471
5521
|
}
|
|
5472
5522
|
} else {
|
|
5473
5523
|
summary[key] = value;
|
|
5474
5524
|
}
|
|
5475
5525
|
}
|
|
5526
|
+
let totalTokens = estimateTokens(summary);
|
|
5527
|
+
if (totalTokens > MAX_STATE_TOKENS) {
|
|
5528
|
+
const collectionSizes = Object.entries(summary).map(([key, value]) => ({ key, tokens: estimateTokens(value) })).sort((a, b) => b.tokens - a.tokens);
|
|
5529
|
+
for (const { key } of collectionSizes) {
|
|
5530
|
+
if (totalTokens <= MAX_STATE_TOKENS) break;
|
|
5531
|
+
const value = summary[key];
|
|
5532
|
+
if (!Array.isArray(value)) continue;
|
|
5533
|
+
const before = estimateTokens(value);
|
|
5534
|
+
summary[key] = {
|
|
5535
|
+
_count: value.length,
|
|
5536
|
+
_first5: value.slice(0, 5),
|
|
5537
|
+
_last5: value.slice(-5),
|
|
5538
|
+
_truncated: "Collection too large for evaluation \u2014 showing subset"
|
|
5539
|
+
};
|
|
5540
|
+
totalTokens -= before - estimateTokens(summary[key]);
|
|
5541
|
+
}
|
|
5542
|
+
}
|
|
5476
5543
|
return summary;
|
|
5477
5544
|
}
|
|
5478
5545
|
function parseJudgeResponse(text) {
|
|
@@ -5572,6 +5639,15 @@ async function evaluateWithLlm(criterion, expectedBehavior, stateBefore, stateAf
|
|
|
5572
5639
|
};
|
|
5573
5640
|
}
|
|
5574
5641
|
const message = err instanceof Error ? err.message : String(err);
|
|
5642
|
+
if (err instanceof LlmApiError && err.status === 400 && message.includes("too long")) {
|
|
5643
|
+
warn(`LLM judge prompt too large for criterion "${criterion.id}" \u2014 twin state may be too large for evaluation`);
|
|
5644
|
+
return {
|
|
5645
|
+
criterionId: criterion.id,
|
|
5646
|
+
status: "fail",
|
|
5647
|
+
confidence: 0,
|
|
5648
|
+
explanation: "LLM evaluation skipped: prompt exceeded model context window. The scenario state is too large for probabilistic evaluation. Consider using deterministic [D] criteria for this scenario."
|
|
5649
|
+
};
|
|
5650
|
+
}
|
|
5575
5651
|
error(`LLM judge call failed: ${message}`);
|
|
5576
5652
|
return {
|
|
5577
5653
|
criterionId: criterion.id,
|
|
@@ -7933,7 +8009,11 @@ function mergeCollectionSchema(entitySchema, overrides) {
|
|
|
7933
8009
|
if (override.default !== void 0) def.default = override.default;
|
|
7934
8010
|
}
|
|
7935
8011
|
if (isRequired) {
|
|
7936
|
-
|
|
8012
|
+
if (nullableSet.has(field)) {
|
|
8013
|
+
def.default = null;
|
|
8014
|
+
} else {
|
|
8015
|
+
delete def.default;
|
|
8016
|
+
}
|
|
7937
8017
|
delete def.required;
|
|
7938
8018
|
}
|
|
7939
8019
|
merged[field] = def;
|
|
@@ -7951,6 +8031,9 @@ function buildSchemaFromOverrides(overrides) {
|
|
|
7951
8031
|
const type = nullableSet.has(field) && !baseType.includes("null") && !baseType.includes("[]") ? `${baseType}|null` : baseType;
|
|
7952
8032
|
schema[field] = {
|
|
7953
8033
|
type,
|
|
8034
|
+
// If field is both required AND nullable, give it a null default so
|
|
8035
|
+
// the LLM can omit it without triggering a hard validation error.
|
|
8036
|
+
...nullableSet.has(field) && { default: null },
|
|
7954
8037
|
...override?.aliases && { aliases: override.aliases },
|
|
7955
8038
|
...override?.description && { description: override.description },
|
|
7956
8039
|
...override?.fk && { fk: override.fk },
|
|
@@ -8063,6 +8146,8 @@ function validateSeedAgainstSchema(twinName, seed, baseEntityCounts) {
|
|
|
8063
8146
|
errors.push(
|
|
8064
8147
|
`${entityLabel}: wrong field name "${usedAlias}" - use "${fieldName}" instead`
|
|
8065
8148
|
);
|
|
8149
|
+
} else if (def.type.includes("null")) {
|
|
8150
|
+
entity[fieldName] = null;
|
|
8066
8151
|
} else {
|
|
8067
8152
|
errors.push(
|
|
8068
8153
|
`${entityLabel}: missing required field "${fieldName}" (${def.type})`
|
|
@@ -8220,7 +8305,8 @@ var RELATIONSHIP_RULES = {
|
|
|
8220
8305
|
{ sourceCollection: "disputes", sourceField: "paymentIntentId", targetCollection: "paymentIntents", targetField: "paymentIntentId", optional: true }
|
|
8221
8306
|
],
|
|
8222
8307
|
jira: [
|
|
8223
|
-
{ sourceCollection: "issues", sourceField: "projectId", targetCollection: "projects", targetField: "id" }
|
|
8308
|
+
{ sourceCollection: "issues", sourceField: "projectId", targetCollection: "projects", targetField: "id" },
|
|
8309
|
+
{ sourceCollection: "projects", sourceField: "leadAccountId", targetCollection: "users", targetField: "accountId" }
|
|
8224
8310
|
],
|
|
8225
8311
|
linear: [
|
|
8226
8312
|
{ sourceCollection: "issues", sourceField: "teamId", targetCollection: "teams", targetField: "id" },
|
|
@@ -8464,15 +8550,17 @@ function autoFillMissingFKs(seed, twinName) {
|
|
|
8464
8550
|
const targetEntities = result[rule.targetCollection];
|
|
8465
8551
|
if (!sourceEntities || !targetEntities || targetEntities.length === 0) continue;
|
|
8466
8552
|
const targetValues = targetEntities.map((e) => e[rule.targetField]).filter((v) => v !== void 0 && v !== null);
|
|
8467
|
-
if (targetValues.length
|
|
8468
|
-
|
|
8553
|
+
if (targetValues.length === 0) continue;
|
|
8554
|
+
let fillIndex = 0;
|
|
8469
8555
|
for (const entity of sourceEntities) {
|
|
8470
8556
|
const e = entity;
|
|
8471
8557
|
if (e[rule.sourceField] === void 0 || e[rule.sourceField] === null) {
|
|
8472
|
-
|
|
8473
|
-
|
|
8558
|
+
const fillValue = targetValues[fillIndex % targetValues.length];
|
|
8559
|
+
fillIndex++;
|
|
8560
|
+
debug(
|
|
8561
|
+
`Auto-filling ${rule.sourceCollection}.${rule.sourceField} = ${String(fillValue)} (from ${targetValues.length} ${rule.targetCollection})`
|
|
8474
8562
|
);
|
|
8475
|
-
e[rule.sourceField] =
|
|
8563
|
+
e[rule.sourceField] = fillValue;
|
|
8476
8564
|
}
|
|
8477
8565
|
}
|
|
8478
8566
|
}
|
|
@@ -8506,12 +8594,36 @@ function normalizeSeedData(seed, twinName) {
|
|
|
8506
8594
|
}
|
|
8507
8595
|
}
|
|
8508
8596
|
}
|
|
8597
|
+
const collectionSchema = schema[collection];
|
|
8598
|
+
if (collectionSchema) {
|
|
8599
|
+
for (const [field, fieldDef] of Object.entries(collectionSchema)) {
|
|
8600
|
+
if (!(field in e) || e[field] === null || e[field] === void 0) continue;
|
|
8601
|
+
const expectedType = fieldDef.type.split("|")[0].trim();
|
|
8602
|
+
if (expectedType === "string" && typeof e[field] === "object" && e[field] !== null && !Array.isArray(e[field])) {
|
|
8603
|
+
const obj = e[field];
|
|
8604
|
+
const extracted = obj["login"] ?? obj["name"] ?? obj["value"] ?? obj["key"] ?? obj["id"] ?? obj["displayName"];
|
|
8605
|
+
if (typeof extracted === "string") {
|
|
8606
|
+
debug(`Seed normalization: coerced ${collection}.${field} from object to string "${extracted}"`);
|
|
8607
|
+
e[field] = extracted;
|
|
8608
|
+
} else {
|
|
8609
|
+
const firstStr = Object.values(obj).find((v) => typeof v === "string");
|
|
8610
|
+
if (firstStr) {
|
|
8611
|
+
debug(`Seed normalization: coerced ${collection}.${field} from object to string "${firstStr}" (fallback)`);
|
|
8612
|
+
e[field] = firstStr;
|
|
8613
|
+
} else {
|
|
8614
|
+
debug(`Seed normalization: could not coerce ${collection}.${field} from object to string, removing`);
|
|
8615
|
+
delete e[field];
|
|
8616
|
+
}
|
|
8617
|
+
}
|
|
8618
|
+
}
|
|
8619
|
+
}
|
|
8620
|
+
}
|
|
8509
8621
|
if (collectionDefaults) {
|
|
8510
8622
|
for (const [field, defaultValue] of Object.entries(collectionDefaults)) {
|
|
8511
8623
|
if (!(field in e)) {
|
|
8512
8624
|
e[field] = structuredClone(defaultValue);
|
|
8513
8625
|
} else if (e[field] === null && defaultValue !== null) {
|
|
8514
|
-
const fieldDef =
|
|
8626
|
+
const fieldDef = collectionSchema?.[field];
|
|
8515
8627
|
if (fieldDef && !fieldDef.type.includes("null")) {
|
|
8516
8628
|
e[field] = structuredClone(defaultValue);
|
|
8517
8629
|
}
|
|
@@ -8520,6 +8632,15 @@ function normalizeSeedData(seed, twinName) {
|
|
|
8520
8632
|
}
|
|
8521
8633
|
}
|
|
8522
8634
|
}
|
|
8635
|
+
if (twinName === "github" && result["repos"]) {
|
|
8636
|
+
for (const entity of result["repos"]) {
|
|
8637
|
+
const e = entity;
|
|
8638
|
+
if ((!e["fullName"] || typeof e["fullName"] !== "string") && typeof e["owner"] === "string" && typeof e["name"] === "string") {
|
|
8639
|
+
e["fullName"] = `${e["owner"]}/${e["name"]}`;
|
|
8640
|
+
debug(`Seed normalization: derived repos.fullName = "${e["fullName"]}"`);
|
|
8641
|
+
}
|
|
8642
|
+
}
|
|
8643
|
+
}
|
|
8523
8644
|
return result;
|
|
8524
8645
|
}
|
|
8525
8646
|
|
|
@@ -8537,7 +8658,6 @@ var KIND_COLLECTION_HINTS = {
|
|
|
8537
8658
|
event: ["events"],
|
|
8538
8659
|
email: ["gmail_messages", "messages"]
|
|
8539
8660
|
};
|
|
8540
|
-
var STRICT_QUOTE_TWINS = /* @__PURE__ */ new Set(["slack", "google-workspace"]);
|
|
8541
8661
|
var ENTITY_KEY_ALIASES = {
|
|
8542
8662
|
"repo.owner": ["ownerLogin", "owner_login", "login", "owner.login", "owner.name"],
|
|
8543
8663
|
"issue.key": ["identifier"],
|
|
@@ -8696,13 +8816,20 @@ function validateSeedCoverage(intent, mergedSeed) {
|
|
|
8696
8816
|
const entityIssues = [];
|
|
8697
8817
|
const quoteErrors = [];
|
|
8698
8818
|
const quoteWarnings = [];
|
|
8819
|
+
const CORE_ENTITY_KEYS = /* @__PURE__ */ new Set(["owner", "name", "fullName", "channel_name", "key", "identifier", "number"]);
|
|
8820
|
+
const entityWarnings = [];
|
|
8699
8821
|
for (const entity of intent.entities) {
|
|
8700
8822
|
if (typeof entity.value === "boolean") continue;
|
|
8701
8823
|
if (!valueExistsInCollections(mergedSeed, entity.kind, entity.key, entity.value)) {
|
|
8702
|
-
|
|
8824
|
+
const issue = {
|
|
8703
8825
|
type: "missing_entity",
|
|
8704
8826
|
message: `Expected ${entity.kind}.${entity.key}=${String(entity.value)} to exist`
|
|
8705
|
-
}
|
|
8827
|
+
};
|
|
8828
|
+
if (CORE_ENTITY_KEYS.has(entity.key)) {
|
|
8829
|
+
entityIssues.push(issue);
|
|
8830
|
+
} else {
|
|
8831
|
+
entityWarnings.push(issue);
|
|
8832
|
+
}
|
|
8706
8833
|
}
|
|
8707
8834
|
}
|
|
8708
8835
|
for (const quote of intent.quotedStrings) {
|
|
@@ -8715,18 +8842,14 @@ function validateSeedCoverage(intent, mergedSeed) {
|
|
|
8715
8842
|
type: "missing_quote",
|
|
8716
8843
|
message: `Expected quoted text to exist: "${quote}"`
|
|
8717
8844
|
};
|
|
8718
|
-
|
|
8719
|
-
quoteErrors.push(issue);
|
|
8720
|
-
} else {
|
|
8721
|
-
quoteWarnings.push(issue);
|
|
8722
|
-
}
|
|
8845
|
+
quoteWarnings.push(issue);
|
|
8723
8846
|
}
|
|
8724
8847
|
}
|
|
8725
8848
|
const errors = [...entityIssues, ...quoteErrors];
|
|
8726
8849
|
return {
|
|
8727
8850
|
valid: errors.length === 0,
|
|
8728
8851
|
issues: errors,
|
|
8729
|
-
warnings: quoteWarnings
|
|
8852
|
+
warnings: [...quoteWarnings, ...entityWarnings]
|
|
8730
8853
|
};
|
|
8731
8854
|
}
|
|
8732
8855
|
|
|
@@ -8794,7 +8917,24 @@ var NON_SUBJECT_STARTS = /* @__PURE__ */ new Set([
|
|
|
8794
8917
|
"could",
|
|
8795
8918
|
"would",
|
|
8796
8919
|
"may",
|
|
8797
|
-
"might"
|
|
8920
|
+
"might",
|
|
8921
|
+
"for",
|
|
8922
|
+
"with",
|
|
8923
|
+
"in",
|
|
8924
|
+
"at",
|
|
8925
|
+
"to",
|
|
8926
|
+
"from",
|
|
8927
|
+
"by",
|
|
8928
|
+
"on",
|
|
8929
|
+
"per",
|
|
8930
|
+
"via",
|
|
8931
|
+
"into",
|
|
8932
|
+
"onto",
|
|
8933
|
+
"over",
|
|
8934
|
+
"under",
|
|
8935
|
+
"after",
|
|
8936
|
+
"before",
|
|
8937
|
+
"during"
|
|
8798
8938
|
]);
|
|
8799
8939
|
function isReasonableCountSubject(subject, expected) {
|
|
8800
8940
|
if (expected > MAX_REASONABLE_COUNT) return false;
|
|
@@ -8805,6 +8945,10 @@ function isReasonableCountSubject(subject, expected) {
|
|
|
8805
8945
|
if (/\b(?:have|has|had|were|was|are|is|been|being|do|does|did|can|could|should|will|would|may|might)\b/.test(subject.toLowerCase())) return false;
|
|
8806
8946
|
return true;
|
|
8807
8947
|
}
|
|
8948
|
+
function appearsToBeClockSuffix(text, numberStart) {
|
|
8949
|
+
const prefix = text.slice(Math.max(0, numberStart - 3), numberStart);
|
|
8950
|
+
return /^\d{1,2}:$/.test(prefix);
|
|
8951
|
+
}
|
|
8808
8952
|
function verifySeedCounts(setupText, seedState) {
|
|
8809
8953
|
const mismatches = [];
|
|
8810
8954
|
const flat = flattenTwinState(seedState);
|
|
@@ -8812,6 +8956,7 @@ function verifySeedCounts(setupText, seedState) {
|
|
|
8812
8956
|
for (const match of setupText.matchAll(countPattern)) {
|
|
8813
8957
|
const expected = parseInt(match[1], 10);
|
|
8814
8958
|
const subject = match[2].trim();
|
|
8959
|
+
if (match.index !== void 0 && appearsToBeClockSuffix(setupText, match.index)) continue;
|
|
8815
8960
|
if (!subject || expected <= 0) continue;
|
|
8816
8961
|
if (!isReasonableCountSubject(subject, expected)) continue;
|
|
8817
8962
|
const resolved = resolveSubjectInState(subject, flat);
|
|
@@ -8824,6 +8969,7 @@ function verifySeedCounts(setupText, seedState) {
|
|
|
8824
8969
|
for (const match of setupText.matchAll(simplePattern)) {
|
|
8825
8970
|
const expected = parseInt(match[1], 10);
|
|
8826
8971
|
const subject = match[2].trim();
|
|
8972
|
+
if (match.index !== void 0 && appearsToBeClockSuffix(setupText, match.index)) continue;
|
|
8827
8973
|
if (!subject || expected <= 0 || seenSubjects.has(subject.toLowerCase())) continue;
|
|
8828
8974
|
if (!isReasonableCountSubject(subject, expected)) continue;
|
|
8829
8975
|
const resolved = resolveSubjectInState(subject, flat);
|
|
@@ -9039,14 +9185,955 @@ function cacheNegativeSeed(twinName, baseSeedName, setupText, missingSlots, scop
|
|
|
9039
9185
|
}
|
|
9040
9186
|
}
|
|
9041
9187
|
|
|
9188
|
+
// src/runner/seed-blueprint.ts
|
|
9189
|
+
var BLUEPRINT_SYSTEM_PROMPT = `You are a precise data extraction tool. Given a scenario setup description, extract a structured JSON blueprint describing what test data needs to be created.
|
|
9190
|
+
|
|
9191
|
+
Output ONLY valid JSON matching this schema:
|
|
9192
|
+
{
|
|
9193
|
+
"identities": [
|
|
9194
|
+
{ "collection": "<collection_name>", "fields": { "<field>": "<value>", ... } }
|
|
9195
|
+
],
|
|
9196
|
+
"collections": [
|
|
9197
|
+
{
|
|
9198
|
+
"name": "<collection_name>",
|
|
9199
|
+
"totalCount": <number>,
|
|
9200
|
+
"groups": [
|
|
9201
|
+
{
|
|
9202
|
+
"count": <number>,
|
|
9203
|
+
"properties": { "<property>": <value>, ... }
|
|
9204
|
+
}
|
|
9205
|
+
],
|
|
9206
|
+
"contentHint": "<optional description of what content should look like>"
|
|
9207
|
+
}
|
|
9208
|
+
]
|
|
9209
|
+
}
|
|
9210
|
+
|
|
9211
|
+
Rules:
|
|
9212
|
+
- "identities" are named entities that MUST exist exactly (repos, channels, users with specific names)
|
|
9213
|
+
- "collections" describe bulk entities with count distributions
|
|
9214
|
+
- Group counts MUST sum to totalCount exactly
|
|
9215
|
+
- Use these property names for temporal attributes:
|
|
9216
|
+
- "stale": true/false (not updated in 90+ days)
|
|
9217
|
+
- "recentlyActive": true/false (updated within 30 days)
|
|
9218
|
+
- Use "state" for entity state: "open", "closed", "merged", etc.
|
|
9219
|
+
- Use "labels" for label arrays: ["bug", "keep-open"]
|
|
9220
|
+
- Use "assigned" for assignment: true/false or assignee name
|
|
9221
|
+
- Use "hasComments": true/false for whether entity has comments
|
|
9222
|
+
- Use "priority" for priority: "P0", "P1", "P2", "critical", "high", "medium", "low"
|
|
9223
|
+
- Use "private" for visibility: true/false
|
|
9224
|
+
- Keep it simple \u2014 only include what the setup text explicitly states
|
|
9225
|
+
- If the setup says "N of those" or "N of the X", that's a subset of the previous group
|
|
9226
|
+
- Do NOT include fields not mentioned in the setup text`;
|
|
9227
|
+
async function extractBlueprint(setupText, twinName, availableCollections, config) {
|
|
9228
|
+
const userPrompt = `Twin: ${twinName}
|
|
9229
|
+
Available collections: ${availableCollections.join(", ")}
|
|
9230
|
+
|
|
9231
|
+
Setup text:
|
|
9232
|
+
${setupText}
|
|
9233
|
+
|
|
9234
|
+
Extract the seed blueprint as JSON.`;
|
|
9235
|
+
try {
|
|
9236
|
+
const provider = detectProvider(config.model);
|
|
9237
|
+
const apiKey = resolveProviderApiKey(config.apiKey, provider);
|
|
9238
|
+
const responseText = await callLlm({
|
|
9239
|
+
provider,
|
|
9240
|
+
model: config.model,
|
|
9241
|
+
apiKey,
|
|
9242
|
+
systemPrompt: BLUEPRINT_SYSTEM_PROMPT,
|
|
9243
|
+
userPrompt,
|
|
9244
|
+
maxTokens: 4096,
|
|
9245
|
+
baseUrl: config.baseUrl,
|
|
9246
|
+
providerMode: config.providerMode,
|
|
9247
|
+
intent: "seed-generate",
|
|
9248
|
+
responseFormat: "json"
|
|
9249
|
+
});
|
|
9250
|
+
if (!responseText) {
|
|
9251
|
+
warn("Blueprint extraction returned no text");
|
|
9252
|
+
return null;
|
|
9253
|
+
}
|
|
9254
|
+
const parsed = parseBlueprint(responseText, twinName);
|
|
9255
|
+
if (!parsed) return null;
|
|
9256
|
+
const validCollections = new Set(availableCollections);
|
|
9257
|
+
parsed.collections = parsed.collections.filter((col) => {
|
|
9258
|
+
if (validCollections.has(col.name)) return true;
|
|
9259
|
+
warn(`Blueprint references unknown collection "${col.name}" for ${twinName} \u2014 dropping`);
|
|
9260
|
+
return false;
|
|
9261
|
+
});
|
|
9262
|
+
for (const col of parsed.collections) {
|
|
9263
|
+
const groupSum = col.groups.reduce((sum, g) => sum + g.count, 0);
|
|
9264
|
+
if (groupSum !== col.totalCount) {
|
|
9265
|
+
debug(`Blueprint group count mismatch for ${col.name}: groups sum to ${groupSum}, totalCount is ${col.totalCount}. Adjusting.`);
|
|
9266
|
+
col.totalCount = groupSum;
|
|
9267
|
+
}
|
|
9268
|
+
if (col.totalCount === 0) {
|
|
9269
|
+
debug(`Blueprint collection ${col.name} has 0 entities \u2014 dropping`);
|
|
9270
|
+
}
|
|
9271
|
+
}
|
|
9272
|
+
parsed.collections = parsed.collections.filter((col) => col.totalCount > 0);
|
|
9273
|
+
if (parsed.collections.length === 0 && parsed.identities.length === 0) {
|
|
9274
|
+
warn("Blueprint extracted no valid collections or identities");
|
|
9275
|
+
return null;
|
|
9276
|
+
}
|
|
9277
|
+
return parsed;
|
|
9278
|
+
} catch (err) {
|
|
9279
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
9280
|
+
warn(`Blueprint extraction failed: ${msg}`);
|
|
9281
|
+
return null;
|
|
9282
|
+
}
|
|
9283
|
+
}
|
|
9284
|
+
function parseBlueprint(text, twinName) {
|
|
9285
|
+
try {
|
|
9286
|
+
let cleaned = text.trim();
|
|
9287
|
+
if (cleaned.startsWith("```")) {
|
|
9288
|
+
cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
|
|
9289
|
+
}
|
|
9290
|
+
const raw = JSON.parse(cleaned);
|
|
9291
|
+
if (!raw || typeof raw !== "object") return null;
|
|
9292
|
+
const identities = [];
|
|
9293
|
+
for (const id of raw.identities ?? []) {
|
|
9294
|
+
if (id?.collection && typeof id.fields === "object") {
|
|
9295
|
+
identities.push({ collection: String(id.collection), fields: id.fields });
|
|
9296
|
+
}
|
|
9297
|
+
}
|
|
9298
|
+
const collections = [];
|
|
9299
|
+
for (const col of raw.collections ?? []) {
|
|
9300
|
+
if (!col?.name || typeof col.totalCount !== "number") continue;
|
|
9301
|
+
const groups = [];
|
|
9302
|
+
for (const g of col.groups ?? []) {
|
|
9303
|
+
if (typeof g?.count === "number" && g.count > 0) {
|
|
9304
|
+
groups.push({ count: g.count, properties: g.properties ?? {} });
|
|
9305
|
+
}
|
|
9306
|
+
}
|
|
9307
|
+
if (groups.length === 0) {
|
|
9308
|
+
groups.push({ count: col.totalCount, properties: {} });
|
|
9309
|
+
}
|
|
9310
|
+
collections.push({
|
|
9311
|
+
name: String(col.name),
|
|
9312
|
+
totalCount: col.totalCount,
|
|
9313
|
+
groups,
|
|
9314
|
+
contentHint: col.contentHint
|
|
9315
|
+
});
|
|
9316
|
+
}
|
|
9317
|
+
return { twin: twinName, identities, collections };
|
|
9318
|
+
} catch {
|
|
9319
|
+
warn("Failed to parse blueprint JSON");
|
|
9320
|
+
return null;
|
|
9321
|
+
}
|
|
9322
|
+
}
|
|
9323
|
+
|
|
9324
|
+
// src/runner/seed-builder.ts
|
|
9325
|
+
var MS_PER_DAY = 864e5;
|
|
9326
|
+
function daysAgo(days, base) {
|
|
9327
|
+
const now = base ?? /* @__PURE__ */ new Date();
|
|
9328
|
+
return new Date(now.getTime() - days * MS_PER_DAY).toISOString();
|
|
9329
|
+
}
|
|
9330
|
+
function recentDate(withinDays = 7, base) {
|
|
9331
|
+
const now = base ?? /* @__PURE__ */ new Date();
|
|
9332
|
+
const offset = Math.floor(Math.random() * withinDays) * MS_PER_DAY;
|
|
9333
|
+
return new Date(now.getTime() - offset).toISOString();
|
|
9334
|
+
}
|
|
9335
|
+
function staleDate(minDays = 91, maxDays = 300, base) {
|
|
9336
|
+
const now = base ?? /* @__PURE__ */ new Date();
|
|
9337
|
+
const range = maxDays - minDays;
|
|
9338
|
+
const offset = (minDays + Math.floor(Math.random() * range)) * MS_PER_DAY;
|
|
9339
|
+
return new Date(now.getTime() - offset).toISOString();
|
|
9340
|
+
}
|
|
9341
|
+
var ISSUE_TITLES = [
|
|
9342
|
+
"Fix login redirect loop on Safari",
|
|
9343
|
+
"Add dark mode support to settings page",
|
|
9344
|
+
"Rate limiter returns 500 instead of 429",
|
|
9345
|
+
"Memory leak in WebSocket connection manager",
|
|
9346
|
+
"Dropdown menus do not close on mobile",
|
|
9347
|
+
"Search results do not highlight matching terms",
|
|
9348
|
+
"File upload fails silently for large files",
|
|
9349
|
+
"Add CSV export to analytics dashboard",
|
|
9350
|
+
"Implement bulk actions for notifications",
|
|
9351
|
+
"GraphQL query returns stale cached data",
|
|
9352
|
+
"Table component lacks keyboard navigation",
|
|
9353
|
+
"Create onboarding tutorial for new members",
|
|
9354
|
+
"API response times degrade for large date ranges",
|
|
9355
|
+
"Add two-factor authentication for admin accounts",
|
|
9356
|
+
"Deprecated crypto API warnings in test suite",
|
|
9357
|
+
"Evaluate replacing Moment.js with date-fns",
|
|
9358
|
+
"Broken tooltip positioning on Safari 17",
|
|
9359
|
+
"Login page shows blank screen on slow 3G",
|
|
9360
|
+
"Intermittent 502 errors on reports endpoint",
|
|
9361
|
+
"Tracking: migrate auth from cookies to JWT",
|
|
9362
|
+
"Roadmap: accessibility audit and WCAG compliance",
|
|
9363
|
+
"Upgrade Node.js runtime from 18 to 20 LTS",
|
|
9364
|
+
"Refactor database connection pool configuration",
|
|
9365
|
+
"Add retry logic for third-party API calls",
|
|
9366
|
+
"Improve error messages for form validation",
|
|
9367
|
+
"Pagination breaks when filtering by status",
|
|
9368
|
+
"Email notification templates are not responsive",
|
|
9369
|
+
"CI pipeline takes 45 minutes on large PRs",
|
|
9370
|
+
"Localization support for date and number formats",
|
|
9371
|
+
"Dashboard widgets do not resize on mobile"
|
|
9372
|
+
];
|
|
9373
|
+
var ISSUE_BODIES = [
|
|
9374
|
+
"This has been reported multiple times by users. Needs investigation and a fix.",
|
|
9375
|
+
"No one has started working on this yet. Low priority but would improve UX.",
|
|
9376
|
+
"This is affecting production traffic. We need to address this soon.",
|
|
9377
|
+
"Users have requested this feature multiple times. Would be a nice enhancement.",
|
|
9378
|
+
"This is a long-running tracking issue. Should remain open for visibility.",
|
|
9379
|
+
"Repro steps: 1) Open the app 2) Navigate to settings 3) Observe the issue.",
|
|
9380
|
+
"This blocks other work. Please prioritize.",
|
|
9381
|
+
"Nice to have. Not urgent but would reduce tech debt."
|
|
9382
|
+
];
|
|
9383
|
+
var PR_TITLES = [
|
|
9384
|
+
"Fix null check in JWT validation",
|
|
9385
|
+
"Add pagination to search results",
|
|
9386
|
+
"Refactor database connection pooling",
|
|
9387
|
+
"Update dependencies to latest versions",
|
|
9388
|
+
"Improve error handling in API middleware",
|
|
9389
|
+
"Add unit tests for auth module",
|
|
9390
|
+
"Fix race condition in WebSocket handler",
|
|
9391
|
+
"Migrate CSS to custom properties for theming"
|
|
9392
|
+
];
|
|
9393
|
+
var CHANNEL_PURPOSES = [
|
|
9394
|
+
"General discussion",
|
|
9395
|
+
"Engineering team updates",
|
|
9396
|
+
"Product announcements",
|
|
9397
|
+
"Customer support escalations",
|
|
9398
|
+
"Random fun stuff",
|
|
9399
|
+
"Incident response",
|
|
9400
|
+
"Design feedback",
|
|
9401
|
+
"Deploy notifications"
|
|
9402
|
+
];
|
|
9403
|
+
var MESSAGE_TEXTS = [
|
|
9404
|
+
"Hey team, just a heads up about the deploy today.",
|
|
9405
|
+
"Can someone review my PR? It's been open for a while.",
|
|
9406
|
+
"The CI pipeline is green again after the fix.",
|
|
9407
|
+
"Meeting notes from today's standup are in the doc.",
|
|
9408
|
+
"Has anyone seen this error before? Getting a 502 intermittently.",
|
|
9409
|
+
"Thanks for the quick turnaround on that bug fix!",
|
|
9410
|
+
"Reminder: retro is at 3pm today.",
|
|
9411
|
+
"I pushed a hotfix for the login issue. Please verify."
|
|
9412
|
+
];
|
|
9413
|
+
function generateNodeId(prefix, id) {
|
|
9414
|
+
return `${prefix}_kgDOB${String(id).padStart(5, "0")}`;
|
|
9415
|
+
}
|
|
9416
|
+
function generateSha() {
|
|
9417
|
+
const hex = "0123456789abcdef";
|
|
9418
|
+
let sha = "";
|
|
9419
|
+
for (let i = 0; i < 40; i++) sha += hex[Math.floor(Math.random() * 16)];
|
|
9420
|
+
return sha;
|
|
9421
|
+
}
|
|
9422
|
+
function generateSlackId(prefix, index) {
|
|
9423
|
+
const base = "ABCDEFG0123456789";
|
|
9424
|
+
let id = prefix;
|
|
9425
|
+
let n = index + 100;
|
|
9426
|
+
for (let i = 0; i < 8; i++) {
|
|
9427
|
+
id += base[n % base.length];
|
|
9428
|
+
n = Math.floor(n / base.length) + i + 1;
|
|
9429
|
+
}
|
|
9430
|
+
return id;
|
|
9431
|
+
}
|
|
9432
|
+
function generateSlackTs(baseTs, index) {
|
|
9433
|
+
return `${baseTs + index * 60}.${String(100001 + index * 100).padStart(6, "0")}`;
|
|
9434
|
+
}
|
|
9435
|
+
function generateUuid() {
|
|
9436
|
+
const hex = "0123456789abcdef";
|
|
9437
|
+
const parts = [8, 4, 4, 4, 12];
|
|
9438
|
+
return parts.map((len) => {
|
|
9439
|
+
let s = "";
|
|
9440
|
+
for (let i = 0; i < len; i++) s += hex[Math.floor(Math.random() * 16)];
|
|
9441
|
+
return s;
|
|
9442
|
+
}).join("-");
|
|
9443
|
+
}
|
|
9444
|
+
function generateStripeId(prefix, index) {
|
|
9445
|
+
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
|
9446
|
+
let id = prefix;
|
|
9447
|
+
let n = index + 42;
|
|
9448
|
+
for (let i = 0; i < 14; i++) {
|
|
9449
|
+
id += chars[n % chars.length];
|
|
9450
|
+
n = Math.floor(n / chars.length) + i + 7;
|
|
9451
|
+
}
|
|
9452
|
+
return id;
|
|
9453
|
+
}
|
|
9454
|
+
var DEFAULT_REACTIONS = {
|
|
9455
|
+
totalCount: 0,
|
|
9456
|
+
plusOne: 0,
|
|
9457
|
+
minusOne: 0,
|
|
9458
|
+
laugh: 0,
|
|
9459
|
+
hooray: 0,
|
|
9460
|
+
confused: 0,
|
|
9461
|
+
heart: 0,
|
|
9462
|
+
rocket: 0,
|
|
9463
|
+
eyes: 0
|
|
9464
|
+
};
|
|
9465
|
+
function resolveTemporalProperties(props, base) {
|
|
9466
|
+
const now = base ?? /* @__PURE__ */ new Date();
|
|
9467
|
+
if (props["stale"] === true) {
|
|
9468
|
+
const created2 = daysAgo(200 + Math.floor(Math.random() * 200), now);
|
|
9469
|
+
const updated2 = staleDate(91, 300, now);
|
|
9470
|
+
return { createdAt: created2, updatedAt: updated2 };
|
|
9471
|
+
}
|
|
9472
|
+
if (props["recentlyActive"] === true) {
|
|
9473
|
+
const created2 = daysAgo(30 + Math.floor(Math.random() * 100), now);
|
|
9474
|
+
const updated2 = recentDate(30, now);
|
|
9475
|
+
return { createdAt: created2, updatedAt: updated2 };
|
|
9476
|
+
}
|
|
9477
|
+
const created = daysAgo(14 + Math.floor(Math.random() * 60), now);
|
|
9478
|
+
const updated = recentDate(14, now);
|
|
9479
|
+
return { createdAt: created, updatedAt: updated };
|
|
9480
|
+
}
|
|
9481
|
+
function resolveLabels(props, availableLabels) {
|
|
9482
|
+
const labels = props["labels"];
|
|
9483
|
+
if (Array.isArray(labels)) return labels.map(String);
|
|
9484
|
+
return [];
|
|
9485
|
+
}
|
|
9486
|
+
function buildSeedFromBlueprint(blueprint, baseSeed) {
|
|
9487
|
+
const seed = structuredClone(baseSeed);
|
|
9488
|
+
const warnings = [];
|
|
9489
|
+
const now = /* @__PURE__ */ new Date();
|
|
9490
|
+
const existingLabels = /* @__PURE__ */ new Set();
|
|
9491
|
+
for (const label of seed["labels"] ?? []) {
|
|
9492
|
+
if (typeof label["name"] === "string") existingLabels.add(label["name"]);
|
|
9493
|
+
}
|
|
9494
|
+
for (const identity of blueprint.identities) {
|
|
9495
|
+
processIdentity(identity, seed, warnings);
|
|
9496
|
+
}
|
|
9497
|
+
const baseCollections = new Set(Object.keys(baseSeed));
|
|
9498
|
+
for (const spec of blueprint.collections) {
|
|
9499
|
+
if (!baseCollections.has(spec.name) && !seed[spec.name]) {
|
|
9500
|
+
warnings.push(`Blueprint references unknown collection "${spec.name}" \u2014 skipping`);
|
|
9501
|
+
warn(`Blueprint references unknown collection "${spec.name}" for ${blueprint.twin} twin \u2014 skipping`);
|
|
9502
|
+
continue;
|
|
9503
|
+
}
|
|
9504
|
+
processCollection(spec, seed, blueprint.twin, existingLabels, warnings, now);
|
|
9505
|
+
}
|
|
9506
|
+
return { seed, warnings };
|
|
9507
|
+
}
|
|
9508
|
+
function processIdentity(identity, seed, warnings) {
|
|
9509
|
+
const collection = identity.collection;
|
|
9510
|
+
if (!seed[collection]) {
|
|
9511
|
+
seed[collection] = [];
|
|
9512
|
+
}
|
|
9513
|
+
const existing = seed[collection].find((entity2) => {
|
|
9514
|
+
for (const [key, value] of Object.entries(identity.fields)) {
|
|
9515
|
+
if (entity2[key] !== value) return false;
|
|
9516
|
+
}
|
|
9517
|
+
return true;
|
|
9518
|
+
});
|
|
9519
|
+
if (existing) {
|
|
9520
|
+
debug(`Identity already exists in ${collection}: ${JSON.stringify(identity.fields)}`);
|
|
9521
|
+
return;
|
|
9522
|
+
}
|
|
9523
|
+
const nextId = getMaxId(seed[collection]) + 1;
|
|
9524
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9525
|
+
const entity = {
|
|
9526
|
+
id: nextId,
|
|
9527
|
+
...identity.fields,
|
|
9528
|
+
createdAt: now,
|
|
9529
|
+
updatedAt: now
|
|
9530
|
+
};
|
|
9531
|
+
fillDefaults(entity, collection);
|
|
9532
|
+
seed[collection].push(entity);
|
|
9533
|
+
debug(`Added identity to ${collection}: ${JSON.stringify(identity.fields)}`);
|
|
9534
|
+
}
|
|
9535
|
+
function processCollection(spec, seed, twinName, existingLabels, warnings, now) {
|
|
9536
|
+
const collection = spec.name;
|
|
9537
|
+
if (!seed[collection]) {
|
|
9538
|
+
seed[collection] = [];
|
|
9539
|
+
}
|
|
9540
|
+
const existingCount = seed[collection].length;
|
|
9541
|
+
if (existingCount > 0) {
|
|
9542
|
+
debug(`Clearing ${existingCount} existing ${collection} entities (blueprint replaces)`);
|
|
9543
|
+
seed[collection] = [];
|
|
9544
|
+
}
|
|
9545
|
+
const referencedLabels = /* @__PURE__ */ new Set();
|
|
9546
|
+
for (const group of spec.groups) {
|
|
9547
|
+
const labels = group.properties["labels"];
|
|
9548
|
+
if (Array.isArray(labels)) {
|
|
9549
|
+
for (const l of labels) {
|
|
9550
|
+
const name = String(l);
|
|
9551
|
+
if (!existingLabels.has(name)) referencedLabels.add(name);
|
|
9552
|
+
}
|
|
9553
|
+
}
|
|
9554
|
+
}
|
|
9555
|
+
if (referencedLabels.size > 0 && seed["labels"]) {
|
|
9556
|
+
const labelColors = ["d73a4a", "a2eeef", "0e8a16", "fbca04", "d876e3", "e4e669", "f9d0c4", "bfdadc"];
|
|
9557
|
+
let labelId = getMaxId(seed["labels"]) + 1;
|
|
9558
|
+
const repoId = findFirstRepoId(seed);
|
|
9559
|
+
for (const name of referencedLabels) {
|
|
9560
|
+
const color = labelColors[labelId % labelColors.length];
|
|
9561
|
+
seed["labels"].push({
|
|
9562
|
+
id: labelId,
|
|
9563
|
+
repoId: repoId ?? 1,
|
|
9564
|
+
nodeId: generateNodeId("LA_kwDOBlab", labelId),
|
|
9565
|
+
name,
|
|
9566
|
+
description: `Label: ${name}`,
|
|
9567
|
+
color,
|
|
9568
|
+
isDefault: false,
|
|
9569
|
+
createdAt: daysAgo(60, now),
|
|
9570
|
+
updatedAt: daysAgo(60, now)
|
|
9571
|
+
});
|
|
9572
|
+
existingLabels.add(name);
|
|
9573
|
+
labelId++;
|
|
9574
|
+
}
|
|
9575
|
+
}
|
|
9576
|
+
let entityIndex = 0;
|
|
9577
|
+
for (const group of spec.groups) {
|
|
9578
|
+
for (let i = 0; i < group.count; i++) {
|
|
9579
|
+
const entity = buildEntity(
|
|
9580
|
+
collection,
|
|
9581
|
+
twinName,
|
|
9582
|
+
group.properties,
|
|
9583
|
+
seed,
|
|
9584
|
+
entityIndex,
|
|
9585
|
+
spec.contentHint,
|
|
9586
|
+
now
|
|
9587
|
+
);
|
|
9588
|
+
seed[collection].push(entity);
|
|
9589
|
+
entityIndex++;
|
|
9590
|
+
}
|
|
9591
|
+
}
|
|
9592
|
+
debug(`Built ${entityIndex} entities for ${collection}`);
|
|
9593
|
+
}
|
|
9594
|
+
function buildEntity(collection, twinName, properties, seed, index, contentHint, now) {
|
|
9595
|
+
const nextId = getMaxId(seed[collection]) + 1;
|
|
9596
|
+
const temporal = resolveTemporalProperties(properties, now);
|
|
9597
|
+
if (twinName === "github") return buildGitHubEntity(collection, nextId, properties, seed, index, temporal, contentHint);
|
|
9598
|
+
if (twinName === "slack") return buildSlackEntity(collection, nextId, properties, seed, index, temporal, contentHint);
|
|
9599
|
+
if (twinName === "linear") return buildLinearEntity(collection, nextId, properties, seed, index, temporal, contentHint);
|
|
9600
|
+
if (twinName === "stripe") return buildStripeEntity(collection, nextId, properties, seed, index, temporal, contentHint);
|
|
9601
|
+
if (twinName === "jira") return buildJiraEntity(collection, nextId, properties, seed, index, temporal, contentHint);
|
|
9602
|
+
return {
|
|
9603
|
+
id: nextId,
|
|
9604
|
+
...properties,
|
|
9605
|
+
createdAt: temporal.createdAt,
|
|
9606
|
+
updatedAt: temporal.updatedAt
|
|
9607
|
+
};
|
|
9608
|
+
}
|
|
9609
|
+
function buildGitHubEntity(collection, id, props, seed, index, temporal, contentHint) {
|
|
9610
|
+
const repoId = findFirstRepoId(seed) ?? 1;
|
|
9611
|
+
const owner = findRepoOwner(seed) ?? "acme";
|
|
9612
|
+
switch (collection) {
|
|
9613
|
+
case "issues": {
|
|
9614
|
+
const state = String(props["state"] ?? "open");
|
|
9615
|
+
const number = getMaxIssueNumber(seed) + 1;
|
|
9616
|
+
const labels = resolveLabels(props, []);
|
|
9617
|
+
const title = ISSUE_TITLES[index % ISSUE_TITLES.length];
|
|
9618
|
+
const body = ISSUE_BODIES[index % ISSUE_BODIES.length];
|
|
9619
|
+
const assigned = props["assigned"] === true;
|
|
9620
|
+
return {
|
|
9621
|
+
id,
|
|
9622
|
+
repoId,
|
|
9623
|
+
nodeId: generateNodeId("I_kwDOBiss", id),
|
|
9624
|
+
number,
|
|
9625
|
+
title: contentHint ? `${title} (${contentHint})` : title,
|
|
9626
|
+
body,
|
|
9627
|
+
state,
|
|
9628
|
+
stateReason: state === "closed" ? "completed" : null,
|
|
9629
|
+
locked: false,
|
|
9630
|
+
assignees: assigned ? [owner] : [],
|
|
9631
|
+
labels,
|
|
9632
|
+
milestone: null,
|
|
9633
|
+
authorLogin: owner,
|
|
9634
|
+
closedAt: state === "closed" ? temporal.updatedAt : null,
|
|
9635
|
+
closedBy: state === "closed" ? owner : null,
|
|
9636
|
+
htmlUrl: `https://github.com/${owner}/webapp/issues/${number}`,
|
|
9637
|
+
isPullRequest: false,
|
|
9638
|
+
reactions: { ...DEFAULT_REACTIONS },
|
|
9639
|
+
createdAt: temporal.createdAt,
|
|
9640
|
+
updatedAt: temporal.updatedAt
|
|
9641
|
+
};
|
|
9642
|
+
}
|
|
9643
|
+
case "pullRequests": {
|
|
9644
|
+
const state = String(props["state"] ?? "open");
|
|
9645
|
+
const number = getMaxPRNumber(seed) + 1;
|
|
9646
|
+
const title = PR_TITLES[index % PR_TITLES.length];
|
|
9647
|
+
const sha = generateSha();
|
|
9648
|
+
const baseSha = generateSha();
|
|
9649
|
+
return {
|
|
9650
|
+
id,
|
|
9651
|
+
repoId,
|
|
9652
|
+
nodeId: generateNodeId("PR_kwDOBpr", id),
|
|
9653
|
+
number,
|
|
9654
|
+
title,
|
|
9655
|
+
body: `This PR addresses ${title.toLowerCase()}.`,
|
|
9656
|
+
state,
|
|
9657
|
+
merged: props["merged"] === true || state === "closed",
|
|
9658
|
+
draft: props["draft"] === true,
|
|
9659
|
+
headRef: `feature/fix-${index}`,
|
|
9660
|
+
headSha: sha,
|
|
9661
|
+
baseRef: "main",
|
|
9662
|
+
baseSha,
|
|
9663
|
+
authorLogin: owner,
|
|
9664
|
+
labels: resolveLabels(props, []),
|
|
9665
|
+
assignees: [],
|
|
9666
|
+
requestedReviewers: [],
|
|
9667
|
+
milestone: null,
|
|
9668
|
+
additions: 10 + index * 5,
|
|
9669
|
+
deletions: 3 + index * 2,
|
|
9670
|
+
changedFiles: 1 + index % 5,
|
|
9671
|
+
commits: 1 + index % 3,
|
|
9672
|
+
comments: 0,
|
|
9673
|
+
reviewComments: 0,
|
|
9674
|
+
maintainerCanModify: true,
|
|
9675
|
+
mergeable: state === "open",
|
|
9676
|
+
mergedAt: props["merged"] === true ? temporal.updatedAt : null,
|
|
9677
|
+
mergedBy: props["merged"] === true ? owner : null,
|
|
9678
|
+
mergeCommitSha: props["merged"] === true ? generateSha() : null,
|
|
9679
|
+
closedAt: state === "closed" ? temporal.updatedAt : null,
|
|
9680
|
+
autoMerge: null,
|
|
9681
|
+
htmlUrl: `https://github.com/${owner}/webapp/pull/${number}`,
|
|
9682
|
+
diffUrl: `https://github.com/${owner}/webapp/pull/${number}.diff`,
|
|
9683
|
+
patchUrl: `https://github.com/${owner}/webapp/pull/${number}.patch`,
|
|
9684
|
+
createdAt: temporal.createdAt,
|
|
9685
|
+
updatedAt: temporal.updatedAt
|
|
9686
|
+
};
|
|
9687
|
+
}
|
|
9688
|
+
case "comments": {
|
|
9689
|
+
const issueNumber = resolveIssueNumberForComment(seed, index);
|
|
9690
|
+
return {
|
|
9691
|
+
id,
|
|
9692
|
+
repoId,
|
|
9693
|
+
nodeId: generateNodeId("IC_kwDOBcom", id),
|
|
9694
|
+
issueNumber,
|
|
9695
|
+
body: props["body"] ? String(props["body"]) : `Comment on issue #${issueNumber}`,
|
|
9696
|
+
authorLogin: owner,
|
|
9697
|
+
authorAssociation: "MEMBER",
|
|
9698
|
+
htmlUrl: `https://github.com/${owner}/webapp/issues/${issueNumber}#issuecomment-${id}`,
|
|
9699
|
+
reactions: { ...DEFAULT_REACTIONS },
|
|
9700
|
+
createdAt: temporal.createdAt,
|
|
9701
|
+
updatedAt: temporal.updatedAt
|
|
9702
|
+
};
|
|
9703
|
+
}
|
|
9704
|
+
case "labels": {
|
|
9705
|
+
const colors = ["d73a4a", "a2eeef", "0e8a16", "fbca04", "d876e3"];
|
|
9706
|
+
return {
|
|
9707
|
+
id,
|
|
9708
|
+
repoId,
|
|
9709
|
+
nodeId: generateNodeId("LA_kwDOBlab", id),
|
|
9710
|
+
name: props["name"] ? String(props["name"]) : `label-${id}`,
|
|
9711
|
+
description: props["description"] ? String(props["description"]) : null,
|
|
9712
|
+
color: colors[id % colors.length],
|
|
9713
|
+
isDefault: false,
|
|
9714
|
+
createdAt: temporal.createdAt,
|
|
9715
|
+
updatedAt: temporal.updatedAt
|
|
9716
|
+
};
|
|
9717
|
+
}
|
|
9718
|
+
default:
|
|
9719
|
+
return {
|
|
9720
|
+
id,
|
|
9721
|
+
repoId,
|
|
9722
|
+
...props,
|
|
9723
|
+
createdAt: temporal.createdAt,
|
|
9724
|
+
updatedAt: temporal.updatedAt
|
|
9725
|
+
};
|
|
9726
|
+
}
|
|
9727
|
+
}
|
|
9728
|
+
function buildSlackEntity(collection, id, props, seed, index, temporal, contentHint) {
|
|
9729
|
+
const teamId = findSlackTeamId(seed) ?? "T0001TEAM";
|
|
9730
|
+
switch (collection) {
|
|
9731
|
+
case "channels": {
|
|
9732
|
+
const channelId = generateSlackId("C", index);
|
|
9733
|
+
const name = props["name"] ? String(props["name"]) : `channel-${index + 1}`;
|
|
9734
|
+
const userIds = (seed["users"] ?? []).map((u) => u["user_id"]).filter(Boolean);
|
|
9735
|
+
return {
|
|
9736
|
+
id,
|
|
9737
|
+
channel_id: channelId,
|
|
9738
|
+
name,
|
|
9739
|
+
is_channel: true,
|
|
9740
|
+
is_group: false,
|
|
9741
|
+
is_im: false,
|
|
9742
|
+
is_mpim: false,
|
|
9743
|
+
is_private: props["private"] === true,
|
|
9744
|
+
is_archived: false,
|
|
9745
|
+
is_general: name === "general",
|
|
9746
|
+
creator: userIds[0] ?? "U0001AAAA",
|
|
9747
|
+
topic: { value: "", creator: "", last_set: 0 },
|
|
9748
|
+
purpose: { value: CHANNEL_PURPOSES[index % CHANNEL_PURPOSES.length], creator: "", last_set: 0 },
|
|
9749
|
+
members: userIds,
|
|
9750
|
+
num_members: userIds.length,
|
|
9751
|
+
created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
|
|
9752
|
+
updated: Math.floor(new Date(temporal.updatedAt).getTime() / 1e3),
|
|
9753
|
+
createdAt: temporal.createdAt,
|
|
9754
|
+
updatedAt: temporal.updatedAt
|
|
9755
|
+
};
|
|
9756
|
+
}
|
|
9757
|
+
case "messages": {
|
|
9758
|
+
const channels = seed["channels"] ?? [];
|
|
9759
|
+
const targetChannel = channels.length > 0 ? channels[index % channels.length] : null;
|
|
9760
|
+
const channelId = targetChannel ? String(targetChannel["channel_id"] ?? "C0001AAAA") : "C0001AAAA";
|
|
9761
|
+
const channelMembers = targetChannel ? targetChannel["members"] ?? [] : [];
|
|
9762
|
+
const users = seed["users"] ?? [];
|
|
9763
|
+
let userId;
|
|
9764
|
+
if (channelMembers.length > 0) {
|
|
9765
|
+
userId = channelMembers[index % channelMembers.length];
|
|
9766
|
+
} else {
|
|
9767
|
+
userId = users.length > 0 ? String(users[index % users.length]["user_id"] ?? "U0001AAAA") : "U0001AAAA";
|
|
9768
|
+
}
|
|
9769
|
+
const baseTs = Math.floor(new Date(temporal.createdAt).getTime() / 1e3);
|
|
9770
|
+
const ts = generateSlackTs(baseTs, index);
|
|
9771
|
+
return {
|
|
9772
|
+
id,
|
|
9773
|
+
ts,
|
|
9774
|
+
channel_id: channelId,
|
|
9775
|
+
user_id: userId,
|
|
9776
|
+
text: props["text"] ? String(props["text"]) : MESSAGE_TEXTS[index % MESSAGE_TEXTS.length],
|
|
9777
|
+
type: "message",
|
|
9778
|
+
subtype: null,
|
|
9779
|
+
thread_ts: props["isReply"] === true ? generateSlackTs(baseTs, 0) : null,
|
|
9780
|
+
reply_count: 0,
|
|
9781
|
+
reply_users: [],
|
|
9782
|
+
reply_users_count: 0,
|
|
9783
|
+
latest_reply: null,
|
|
9784
|
+
createdAt: temporal.createdAt,
|
|
9785
|
+
updatedAt: temporal.updatedAt
|
|
9786
|
+
};
|
|
9787
|
+
}
|
|
9788
|
+
case "users": {
|
|
9789
|
+
const userId = generateSlackId("U", index);
|
|
9790
|
+
const name = props["name"] ? String(props["name"]) : `user${index + 1}`;
|
|
9791
|
+
return {
|
|
9792
|
+
id,
|
|
9793
|
+
user_id: userId,
|
|
9794
|
+
team_id: teamId,
|
|
9795
|
+
name,
|
|
9796
|
+
real_name: props["real_name"] ? String(props["real_name"]) : `User ${index + 1}`,
|
|
9797
|
+
display_name: name,
|
|
9798
|
+
email: `${name}@example.com`,
|
|
9799
|
+
is_admin: props["is_admin"] === true,
|
|
9800
|
+
is_owner: false,
|
|
9801
|
+
is_bot: props["is_bot"] === true,
|
|
9802
|
+
is_restricted: false,
|
|
9803
|
+
is_ultra_restricted: false,
|
|
9804
|
+
deleted: false,
|
|
9805
|
+
color: "4bbe2e",
|
|
9806
|
+
timezone: "America/Los_Angeles",
|
|
9807
|
+
tz_label: "Pacific Daylight Time",
|
|
9808
|
+
tz_offset: -25200,
|
|
9809
|
+
is_email_confirmed: true,
|
|
9810
|
+
who_can_share_contact_card: "EVERYONE",
|
|
9811
|
+
createdAt: temporal.createdAt,
|
|
9812
|
+
updatedAt: temporal.updatedAt
|
|
9813
|
+
};
|
|
9814
|
+
}
|
|
9815
|
+
default:
|
|
9816
|
+
return {
|
|
9817
|
+
id,
|
|
9818
|
+
...props,
|
|
9819
|
+
createdAt: temporal.createdAt,
|
|
9820
|
+
updatedAt: temporal.updatedAt
|
|
9821
|
+
};
|
|
9822
|
+
}
|
|
9823
|
+
}
|
|
9824
|
+
function buildLinearEntity(collection, id, props, seed, index, temporal, contentHint) {
|
|
9825
|
+
switch (collection) {
|
|
9826
|
+
case "issues": {
|
|
9827
|
+
const teamId = findFirstId(seed, "teams") ?? 1;
|
|
9828
|
+
const team = (seed["teams"] ?? []).find((t) => t["id"] === teamId);
|
|
9829
|
+
const teamKey = team ? String(team["key"] ?? "ENG") : "ENG";
|
|
9830
|
+
const number = index + 1;
|
|
9831
|
+
const stateId = resolveLinearStateId(seed, props);
|
|
9832
|
+
const priority = typeof props["priority"] === "number" ? props["priority"] : 3;
|
|
9833
|
+
const priorityMap = {
|
|
9834
|
+
0: "No priority",
|
|
9835
|
+
1: "Urgent",
|
|
9836
|
+
2: "High",
|
|
9837
|
+
3: "Medium",
|
|
9838
|
+
4: "Low"
|
|
9839
|
+
};
|
|
9840
|
+
return {
|
|
9841
|
+
id,
|
|
9842
|
+
linearId: generateUuid(),
|
|
9843
|
+
teamId,
|
|
9844
|
+
number,
|
|
9845
|
+
identifier: `${teamKey}-${number}`,
|
|
9846
|
+
title: ISSUE_TITLES[index % ISSUE_TITLES.length],
|
|
9847
|
+
description: null,
|
|
9848
|
+
descriptionData: null,
|
|
9849
|
+
priority,
|
|
9850
|
+
priorityLabel: priorityMap[priority] ?? "Medium",
|
|
9851
|
+
stateId,
|
|
9852
|
+
assigneeId: null,
|
|
9853
|
+
creatorId: null,
|
|
9854
|
+
projectId: null,
|
|
9855
|
+
cycleId: null,
|
|
9856
|
+
parentId: null,
|
|
9857
|
+
labelIds: [],
|
|
9858
|
+
estimate: null,
|
|
9859
|
+
dueDate: null,
|
|
9860
|
+
subIssueSortOrder: null,
|
|
9861
|
+
sortOrder: index * -100,
|
|
9862
|
+
snoozedUntilAt: null,
|
|
9863
|
+
startedAt: null,
|
|
9864
|
+
completedAt: null,
|
|
9865
|
+
canceledAt: null,
|
|
9866
|
+
archivedAt: null,
|
|
9867
|
+
trashed: false,
|
|
9868
|
+
createdAt: temporal.createdAt,
|
|
9869
|
+
updatedAt: temporal.updatedAt
|
|
9870
|
+
};
|
|
9871
|
+
}
|
|
9872
|
+
default:
|
|
9873
|
+
return {
|
|
9874
|
+
id,
|
|
9875
|
+
linearId: generateUuid(),
|
|
9876
|
+
...props,
|
|
9877
|
+
createdAt: temporal.createdAt,
|
|
9878
|
+
updatedAt: temporal.updatedAt
|
|
9879
|
+
};
|
|
9880
|
+
}
|
|
9881
|
+
}
|
|
9882
|
+
function buildStripeEntity(collection, id, props, seed, index, temporal, contentHint) {
|
|
9883
|
+
switch (collection) {
|
|
9884
|
+
case "paymentIntents": {
|
|
9885
|
+
const piId = generateStripeId("pi_", index);
|
|
9886
|
+
const amount = typeof props["amount"] === "number" ? props["amount"] : 1e3 + index * 500;
|
|
9887
|
+
const status = String(props["status"] ?? "succeeded");
|
|
9888
|
+
return {
|
|
9889
|
+
id,
|
|
9890
|
+
paymentIntentId: piId,
|
|
9891
|
+
amount,
|
|
9892
|
+
currency: String(props["currency"] ?? "usd"),
|
|
9893
|
+
status,
|
|
9894
|
+
customerId: props["customerId"] ? String(props["customerId"]) : null,
|
|
9895
|
+
description: props["description"] ? String(props["description"]) : null,
|
|
9896
|
+
paymentMethodId: null,
|
|
9897
|
+
captureMethod: "automatic",
|
|
9898
|
+
confirmationMethod: "automatic",
|
|
9899
|
+
canceledAt: null,
|
|
9900
|
+
cancellationReason: null,
|
|
9901
|
+
latestChargeId: null,
|
|
9902
|
+
metadata: {},
|
|
9903
|
+
livemode: false,
|
|
9904
|
+
created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
|
|
9905
|
+
createdAt: temporal.createdAt,
|
|
9906
|
+
updatedAt: temporal.updatedAt
|
|
9907
|
+
};
|
|
9908
|
+
}
|
|
9909
|
+
case "customers": {
|
|
9910
|
+
const custId = generateStripeId("cus_", index);
|
|
9911
|
+
return {
|
|
9912
|
+
id,
|
|
9913
|
+
customerId: custId,
|
|
9914
|
+
name: props["name"] ? String(props["name"]) : `Customer ${index + 1}`,
|
|
9915
|
+
email: props["email"] ? String(props["email"]) : `customer${index + 1}@example.com`,
|
|
9916
|
+
phone: null,
|
|
9917
|
+
description: null,
|
|
9918
|
+
currency: null,
|
|
9919
|
+
defaultPaymentMethod: null,
|
|
9920
|
+
address: null,
|
|
9921
|
+
shipping: null,
|
|
9922
|
+
balance: 0,
|
|
9923
|
+
delinquent: false,
|
|
9924
|
+
metadata: {},
|
|
9925
|
+
livemode: false,
|
|
9926
|
+
created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
|
|
9927
|
+
createdAt: temporal.createdAt,
|
|
9928
|
+
updatedAt: temporal.updatedAt
|
|
9929
|
+
};
|
|
9930
|
+
}
|
|
9931
|
+
default:
|
|
9932
|
+
return {
|
|
9933
|
+
id,
|
|
9934
|
+
...props,
|
|
9935
|
+
created: Math.floor(new Date(temporal.createdAt).getTime() / 1e3),
|
|
9936
|
+
createdAt: temporal.createdAt,
|
|
9937
|
+
updatedAt: temporal.updatedAt
|
|
9938
|
+
};
|
|
9939
|
+
}
|
|
9940
|
+
}
|
|
9941
|
+
function buildJiraEntity(collection, id, props, seed, index, temporal, contentHint) {
|
|
9942
|
+
switch (collection) {
|
|
9943
|
+
case "issues": {
|
|
9944
|
+
const projectId = findFirstId(seed, "projects") ?? 1;
|
|
9945
|
+
const project = (seed["projects"] ?? []).find((p) => p["id"] === projectId);
|
|
9946
|
+
const projectKey = project ? String(project["key"] ?? "PROJ") : "PROJ";
|
|
9947
|
+
const number = index + 1;
|
|
9948
|
+
return {
|
|
9949
|
+
id,
|
|
9950
|
+
key: `${projectKey}-${number}`,
|
|
9951
|
+
projectId,
|
|
9952
|
+
issueTypeId: findFirstId(seed, "issueTypes") ?? 1,
|
|
9953
|
+
summary: ISSUE_TITLES[index % ISSUE_TITLES.length],
|
|
9954
|
+
description: null,
|
|
9955
|
+
statusId: findFirstId(seed, "statuses") ?? 1,
|
|
9956
|
+
priorityId: findFirstId(seed, "priorities") ?? 1,
|
|
9957
|
+
reporterAccountId: findFirstAccountId(seed),
|
|
9958
|
+
assigneeAccountId: props["assigned"] === true ? findFirstAccountId(seed) : null,
|
|
9959
|
+
parentKey: null,
|
|
9960
|
+
storyPoints: null,
|
|
9961
|
+
resolution: null,
|
|
9962
|
+
resolutionDate: null,
|
|
9963
|
+
labels: resolveLabels(props, []),
|
|
9964
|
+
components: [],
|
|
9965
|
+
fixVersions: [],
|
|
9966
|
+
createdAt: temporal.createdAt,
|
|
9967
|
+
updatedAt: temporal.updatedAt
|
|
9968
|
+
};
|
|
9969
|
+
}
|
|
9970
|
+
default:
|
|
9971
|
+
return {
|
|
9972
|
+
id,
|
|
9973
|
+
...props,
|
|
9974
|
+
createdAt: temporal.createdAt,
|
|
9975
|
+
updatedAt: temporal.updatedAt
|
|
9976
|
+
};
|
|
9977
|
+
}
|
|
9978
|
+
}
|
|
9979
|
+
function findFirstRepoId(seed) {
|
|
9980
|
+
const repos = seed["repos"];
|
|
9981
|
+
return repos?.[0]?.["id"];
|
|
9982
|
+
}
|
|
9983
|
+
function findRepoOwner(seed) {
|
|
9984
|
+
const repos = seed["repos"];
|
|
9985
|
+
return repos?.[0]?.["owner"];
|
|
9986
|
+
}
|
|
9987
|
+
function findFirstId(seed, collection) {
|
|
9988
|
+
const entities = seed[collection];
|
|
9989
|
+
return entities?.[0]?.["id"];
|
|
9990
|
+
}
|
|
9991
|
+
function findFirstAccountId(seed) {
|
|
9992
|
+
const users = seed["users"];
|
|
9993
|
+
return String(users?.[0]?.["accountId"] ?? "account-1");
|
|
9994
|
+
}
|
|
9995
|
+
function findSlackTeamId(seed) {
|
|
9996
|
+
const workspaces = seed["workspaces"];
|
|
9997
|
+
if (workspaces?.[0]) return String(workspaces[0]["team_id"]);
|
|
9998
|
+
const users = seed["users"];
|
|
9999
|
+
if (users?.[0]) return String(users[0]["team_id"] ?? "T0001TEAM");
|
|
10000
|
+
return void 0;
|
|
10001
|
+
}
|
|
10002
|
+
function getMaxIssueNumber(seed) {
|
|
10003
|
+
let max = 0;
|
|
10004
|
+
for (const issue of seed["issues"] ?? []) {
|
|
10005
|
+
const n = issue["number"];
|
|
10006
|
+
if (typeof n === "number" && n > max) max = n;
|
|
10007
|
+
}
|
|
10008
|
+
return max;
|
|
10009
|
+
}
|
|
10010
|
+
function getMaxPRNumber(seed) {
|
|
10011
|
+
let max = 0;
|
|
10012
|
+
for (const pr of seed["pullRequests"] ?? []) {
|
|
10013
|
+
const n = pr["number"];
|
|
10014
|
+
if (typeof n === "number" && n > max) max = n;
|
|
10015
|
+
}
|
|
10016
|
+
return Math.max(max, getMaxIssueNumber(seed));
|
|
10017
|
+
}
|
|
10018
|
+
function resolveIssueNumberForComment(seed, index) {
|
|
10019
|
+
const issues = seed["issues"] ?? [];
|
|
10020
|
+
if (issues.length === 0) return 1;
|
|
10021
|
+
const issue = issues[index % issues.length];
|
|
10022
|
+
return issue["number"] ?? 1;
|
|
10023
|
+
}
|
|
10024
|
+
function resolveLinearStateId(seed, props) {
|
|
10025
|
+
const states = seed["workflowStates"] ?? [];
|
|
10026
|
+
if (states.length === 0) return 1;
|
|
10027
|
+
const state = props["state"];
|
|
10028
|
+
if (state === "completed" || state === "closed" || state === "done") {
|
|
10029
|
+
const found = states.find((s) => s["type"] === "completed");
|
|
10030
|
+
if (found) return found["id"];
|
|
10031
|
+
}
|
|
10032
|
+
if (state === "cancelled" || state === "canceled") {
|
|
10033
|
+
const found = states.find((s) => s["type"] === "cancelled");
|
|
10034
|
+
if (found) return found["id"];
|
|
10035
|
+
}
|
|
10036
|
+
if (state === "in_progress" || state === "started") {
|
|
10037
|
+
const found = states.find((s) => s["type"] === "started");
|
|
10038
|
+
if (found) return found["id"];
|
|
10039
|
+
}
|
|
10040
|
+
if (state === "backlog") {
|
|
10041
|
+
const found = states.find((s) => s["type"] === "backlog");
|
|
10042
|
+
if (found) return found["id"];
|
|
10043
|
+
}
|
|
10044
|
+
const unstarted = states.find((s) => s["type"] === "unstarted");
|
|
10045
|
+
return unstarted?.["id"] ?? states[0]["id"];
|
|
10046
|
+
}
|
|
10047
|
+
function fillDefaults(entity, collection) {
|
|
10048
|
+
switch (collection) {
|
|
10049
|
+
case "repos":
|
|
10050
|
+
entity["fullName"] = entity["fullName"] ?? `${entity["owner"] ?? "org"}/${entity["name"] ?? "repo"}`;
|
|
10051
|
+
entity["private"] = entity["private"] ?? false;
|
|
10052
|
+
entity["fork"] = entity["fork"] ?? false;
|
|
10053
|
+
entity["defaultBranch"] = entity["defaultBranch"] ?? "main";
|
|
10054
|
+
entity["archived"] = entity["archived"] ?? false;
|
|
10055
|
+
entity["disabled"] = entity["disabled"] ?? false;
|
|
10056
|
+
entity["visibility"] = entity["visibility"] ?? "public";
|
|
10057
|
+
entity["hasIssues"] = entity["hasIssues"] ?? true;
|
|
10058
|
+
entity["topics"] = entity["topics"] ?? [];
|
|
10059
|
+
entity["openIssuesCount"] = entity["openIssuesCount"] ?? 0;
|
|
10060
|
+
entity["stargazersCount"] = entity["stargazersCount"] ?? 0;
|
|
10061
|
+
entity["forksCount"] = entity["forksCount"] ?? 0;
|
|
10062
|
+
entity["watchersCount"] = entity["watchersCount"] ?? 0;
|
|
10063
|
+
break;
|
|
10064
|
+
case "users":
|
|
10065
|
+
entity["type"] = entity["type"] ?? "User";
|
|
10066
|
+
entity["siteAdmin"] = entity["siteAdmin"] ?? false;
|
|
10067
|
+
entity["publicRepos"] = entity["publicRepos"] ?? 0;
|
|
10068
|
+
entity["followers"] = entity["followers"] ?? 0;
|
|
10069
|
+
entity["following"] = entity["following"] ?? 0;
|
|
10070
|
+
break;
|
|
10071
|
+
}
|
|
10072
|
+
}
|
|
10073
|
+
|
|
9042
10074
|
// src/runner/dynamic-seed-generator.ts
|
|
10075
|
+
function extractCountRequirements(setupText) {
|
|
10076
|
+
const requirements = [];
|
|
10077
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10078
|
+
const NON_ENTITY = /* @__PURE__ */ new Set([
|
|
10079
|
+
"minutes",
|
|
10080
|
+
"minute",
|
|
10081
|
+
"hours",
|
|
10082
|
+
"hour",
|
|
10083
|
+
"days",
|
|
10084
|
+
"day",
|
|
10085
|
+
"weeks",
|
|
10086
|
+
"week",
|
|
10087
|
+
"months",
|
|
10088
|
+
"month",
|
|
10089
|
+
"years",
|
|
10090
|
+
"year",
|
|
10091
|
+
"seconds",
|
|
10092
|
+
"second",
|
|
10093
|
+
"ms",
|
|
10094
|
+
"am",
|
|
10095
|
+
"pm",
|
|
10096
|
+
"st",
|
|
10097
|
+
"nd",
|
|
10098
|
+
"rd",
|
|
10099
|
+
"th",
|
|
10100
|
+
"usd",
|
|
10101
|
+
"eur",
|
|
10102
|
+
"gbp",
|
|
10103
|
+
"percent",
|
|
10104
|
+
"kb",
|
|
10105
|
+
"mb",
|
|
10106
|
+
"gb",
|
|
10107
|
+
"tb"
|
|
10108
|
+
]);
|
|
10109
|
+
const patterns = [
|
|
10110
|
+
/\b(\d+)\s+([\w\s]+?)(?:\s+(?:that|which|are|with|in|labeled|assigned|have|has)\b)/gi,
|
|
10111
|
+
/\b(\d+)\s+([\w\s]+?)(?:[.,;:)]|$)/gm
|
|
10112
|
+
];
|
|
10113
|
+
for (const pattern of patterns) {
|
|
10114
|
+
for (const match of setupText.matchAll(pattern)) {
|
|
10115
|
+
const count = parseInt(match[1], 10);
|
|
10116
|
+
const subject = match[2].trim().toLowerCase();
|
|
10117
|
+
if (!subject || count <= 0 || count > 200) continue;
|
|
10118
|
+
const firstWord = subject.split(/\s+/)[0] ?? "";
|
|
10119
|
+
if (NON_ENTITY.has(firstWord)) continue;
|
|
10120
|
+
if (/^(?:of|and|or|the|those|these|them)\b/.test(subject)) continue;
|
|
10121
|
+
const key = `${count}:${subject}`;
|
|
10122
|
+
if (seen.has(key)) continue;
|
|
10123
|
+
seen.add(key);
|
|
10124
|
+
requirements.push(`- ${subject}: exactly ${count}`);
|
|
10125
|
+
}
|
|
10126
|
+
}
|
|
10127
|
+
return requirements;
|
|
10128
|
+
}
|
|
9043
10129
|
var DynamicSeedError = class extends Error {
|
|
9044
10130
|
twinName;
|
|
9045
10131
|
validationErrors;
|
|
9046
10132
|
constructor(twinName, validationErrors) {
|
|
9047
10133
|
const details = validationErrors.length > 0 ? `:
|
|
9048
10134
|
${validationErrors.map((e) => ` - ${e}`).join("\n")}` : ".";
|
|
9049
|
-
|
|
10135
|
+
const hint = "\n\nHint: Try `--static-seed` to skip dynamic generation and use pre-built seeds instead.";
|
|
10136
|
+
super(`Dynamic seed generation failed for twin "${twinName}"${details}${hint}`);
|
|
9050
10137
|
this.name = "DynamicSeedError";
|
|
9051
10138
|
this.twinName = twinName;
|
|
9052
10139
|
this.validationErrors = validationErrors;
|
|
@@ -9280,6 +10367,14 @@ ${context.expectedBehavior}
|
|
|
9280
10367
|
prompt += context.successCriteria.map((criterion) => `- ${criterion}`).join("\n");
|
|
9281
10368
|
prompt += "\n\n";
|
|
9282
10369
|
}
|
|
10370
|
+
const countReqs = extractCountRequirements(setupDescription);
|
|
10371
|
+
if (countReqs.length > 0) {
|
|
10372
|
+
prompt += `## MANDATORY Count Requirements
|
|
10373
|
+
The final seed state MUST have these exact counts. Do NOT deviate:
|
|
10374
|
+
`;
|
|
10375
|
+
prompt += countReqs.join("\n");
|
|
10376
|
+
prompt += "\n\n";
|
|
10377
|
+
}
|
|
9283
10378
|
prompt += `## Setup Description
|
|
9284
10379
|
This is a multi-service scenario. Generate ONLY the ${twinName} seed data. Ignore setup details that belong to other services. Faithfully reproduce EVERY detail relevant to ${twinName}. Specific names, messages, amounts, and entities mentioned MUST exist in the generated data.
|
|
9285
10380
|
|
|
@@ -9682,6 +10777,87 @@ function parseSeedPatchResponse(text, twinName) {
|
|
|
9682
10777
|
});
|
|
9683
10778
|
return null;
|
|
9684
10779
|
}
|
|
10780
|
+
async function tryBlueprintPath(twinName, baseSeedData, setupDescription, availableCollections, config, intent) {
|
|
10781
|
+
try {
|
|
10782
|
+
const blueprintConfig = {
|
|
10783
|
+
apiKey: config.apiKey,
|
|
10784
|
+
model: config.model,
|
|
10785
|
+
baseUrl: config.baseUrl,
|
|
10786
|
+
providerMode: config.providerMode
|
|
10787
|
+
};
|
|
10788
|
+
const blueprint = await extractBlueprint(
|
|
10789
|
+
setupDescription,
|
|
10790
|
+
twinName,
|
|
10791
|
+
availableCollections,
|
|
10792
|
+
blueprintConfig
|
|
10793
|
+
);
|
|
10794
|
+
if (!blueprint) {
|
|
10795
|
+
debug("Blueprint extraction returned null");
|
|
10796
|
+
return null;
|
|
10797
|
+
}
|
|
10798
|
+
if (blueprint.collections.length === 0 && blueprint.identities.length === 0) {
|
|
10799
|
+
debug("Blueprint extraction returned empty blueprint");
|
|
10800
|
+
return null;
|
|
10801
|
+
}
|
|
10802
|
+
debug("Blueprint extracted", {
|
|
10803
|
+
identities: String(blueprint.identities.length),
|
|
10804
|
+
collections: blueprint.collections.map((c) => `${c.name}:${c.totalCount}`).join(", ")
|
|
10805
|
+
});
|
|
10806
|
+
const { seed: builtSeed, warnings } = buildSeedFromBlueprint(blueprint, baseSeedData);
|
|
10807
|
+
if (warnings.length > 0) {
|
|
10808
|
+
debug("Blueprint builder warnings", { warnings: warnings.join("; ") });
|
|
10809
|
+
}
|
|
10810
|
+
let finalSeed = normalizeSeedData(builtSeed, twinName);
|
|
10811
|
+
finalSeed = autoFillMissingFKs(finalSeed, twinName);
|
|
10812
|
+
const relValidation = validateSeedRelationships(finalSeed, twinName);
|
|
10813
|
+
if (!relValidation.valid) {
|
|
10814
|
+
warn("Blueprint seed failed relationship validation", {
|
|
10815
|
+
errors: relValidation.errors.slice(0, 5).join("; ")
|
|
10816
|
+
});
|
|
10817
|
+
return null;
|
|
10818
|
+
}
|
|
10819
|
+
if (intent) {
|
|
10820
|
+
const coverage = validateSeedCoverage(intent, finalSeed);
|
|
10821
|
+
if (!coverage.valid) {
|
|
10822
|
+
warn("Blueprint seed failed coverage validation", {
|
|
10823
|
+
errors: coverage.issues.map((i) => i.message).join("; ")
|
|
10824
|
+
});
|
|
10825
|
+
return null;
|
|
10826
|
+
}
|
|
10827
|
+
}
|
|
10828
|
+
const flatForVerify = {};
|
|
10829
|
+
flatForVerify[twinName] = finalSeed;
|
|
10830
|
+
const countMismatches = verifySeedCounts(setupDescription, flatForVerify);
|
|
10831
|
+
if (countMismatches.length > 0) {
|
|
10832
|
+
debug("Blueprint seed has count mismatches (acceptable)", {
|
|
10833
|
+
mismatches: countMismatches.map((m) => `${m.subject}: ${m.expected} vs ${m.actual}`).join("; ")
|
|
10834
|
+
});
|
|
10835
|
+
}
|
|
10836
|
+
const syntheticPatch = {
|
|
10837
|
+
add: {}
|
|
10838
|
+
};
|
|
10839
|
+
for (const [collection, entities] of Object.entries(finalSeed)) {
|
|
10840
|
+
const baseCount = baseSeedData[collection]?.length ?? 0;
|
|
10841
|
+
if (entities.length > baseCount) {
|
|
10842
|
+
syntheticPatch.add[collection] = entities.slice(baseCount).map((e) => {
|
|
10843
|
+
const { id: _id, createdAt: _ca, updatedAt: _ua, ...rest } = e;
|
|
10844
|
+
return rest;
|
|
10845
|
+
});
|
|
10846
|
+
}
|
|
10847
|
+
}
|
|
10848
|
+
return {
|
|
10849
|
+
seed: finalSeed,
|
|
10850
|
+
patch: syntheticPatch,
|
|
10851
|
+
fromCache: false,
|
|
10852
|
+
source: "llm"
|
|
10853
|
+
// Blueprint still uses LLM for extraction
|
|
10854
|
+
};
|
|
10855
|
+
} catch (err) {
|
|
10856
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
10857
|
+
warn(`Blueprint path failed: ${msg}`);
|
|
10858
|
+
return null;
|
|
10859
|
+
}
|
|
10860
|
+
}
|
|
9685
10861
|
async function generateDynamicSeed(twinName, baseSeedName, baseSeedData, setupDescription, config, intent, context) {
|
|
9686
10862
|
const cacheScope = {
|
|
9687
10863
|
baseSeedData,
|
|
@@ -9703,6 +10879,28 @@ async function generateDynamicSeed(twinName, baseSeedName, baseSeedData, setupDe
|
|
|
9703
10879
|
"No API key configured for seed generation. Set ARCHAL_TOKEN or configure a provider API key."
|
|
9704
10880
|
]);
|
|
9705
10881
|
}
|
|
10882
|
+
progress(`Generating dynamic seed for ${twinName}...`);
|
|
10883
|
+
const availableCollections = Object.keys(baseSeedData);
|
|
10884
|
+
const blueprintResult = await tryBlueprintPath(
|
|
10885
|
+
twinName,
|
|
10886
|
+
baseSeedData,
|
|
10887
|
+
setupDescription,
|
|
10888
|
+
availableCollections,
|
|
10889
|
+
config,
|
|
10890
|
+
intent
|
|
10891
|
+
);
|
|
10892
|
+
if (blueprintResult) {
|
|
10893
|
+
info("Dynamic seed generated via blueprint", { twin: twinName });
|
|
10894
|
+
if (!config.noCache) {
|
|
10895
|
+
const cacheContext = buildSeedCacheContext(twinName, intent, context);
|
|
10896
|
+
cacheSeed(twinName, baseSeedName, setupDescription, blueprintResult.seed, blueprintResult.patch, {
|
|
10897
|
+
baseSeedData,
|
|
10898
|
+
cacheContext
|
|
10899
|
+
});
|
|
10900
|
+
}
|
|
10901
|
+
return blueprintResult;
|
|
10902
|
+
}
|
|
10903
|
+
debug("Blueprint path failed or produced invalid seed, falling back to full LLM generation");
|
|
9706
10904
|
const userPrompt = buildSeedGenerationPrompt(
|
|
9707
10905
|
twinName,
|
|
9708
10906
|
baseSeedData,
|
|
@@ -9710,7 +10908,6 @@ async function generateDynamicSeed(twinName, baseSeedName, baseSeedData, setupDe
|
|
|
9710
10908
|
intent,
|
|
9711
10909
|
context
|
|
9712
10910
|
);
|
|
9713
|
-
progress(`Generating dynamic seed for ${twinName}...`);
|
|
9714
10911
|
let patch = null;
|
|
9715
10912
|
let mergedSeed = null;
|
|
9716
10913
|
let lastErrors = [];
|
|
@@ -9741,7 +10938,7 @@ Fix these issues:
|
|
|
9741
10938
|
validationAttempt: String(validationAttempts + 1)
|
|
9742
10939
|
});
|
|
9743
10940
|
const provider = detectProvider(config.model);
|
|
9744
|
-
const apiKey = resolveProviderApiKey(config.apiKey, provider);
|
|
10941
|
+
const apiKey = effectiveMode === "archal" ? "" : resolveProviderApiKey(config.apiKey, provider);
|
|
9745
10942
|
const responseText = await callLlm({
|
|
9746
10943
|
provider,
|
|
9747
10944
|
model: config.model,
|
|
@@ -9750,7 +10947,7 @@ Fix these issues:
|
|
|
9750
10947
|
userPrompt: promptWithFeedback,
|
|
9751
10948
|
maxTokens: 16384,
|
|
9752
10949
|
baseUrl: config.baseUrl,
|
|
9753
|
-
providerMode:
|
|
10950
|
+
providerMode: effectiveMode,
|
|
9754
10951
|
intent: "seed-generate",
|
|
9755
10952
|
responseFormat: "json"
|
|
9756
10953
|
});
|
|
@@ -9860,6 +11057,22 @@ Fix these issues:
|
|
|
9860
11057
|
continue;
|
|
9861
11058
|
}
|
|
9862
11059
|
}
|
|
11060
|
+
if (mergedSeed && setupDescription && validationAttempts < MAX_ATTEMPTS - 1) {
|
|
11061
|
+
const flatForVerify = {};
|
|
11062
|
+
flatForVerify[twinName] = mergedSeed;
|
|
11063
|
+
const countMismatches = verifySeedCounts(setupDescription, flatForVerify);
|
|
11064
|
+
if (countMismatches.length > 0) {
|
|
11065
|
+
const countErrors = countMismatches.map(
|
|
11066
|
+
(m) => `Count mismatch: "${m.subject}" should be ${m.expected} but got ${m.actual}`
|
|
11067
|
+
);
|
|
11068
|
+
warn(`Seed count mismatch (attempt ${attempt + 1})`, {
|
|
11069
|
+
errors: countErrors.join("; ")
|
|
11070
|
+
});
|
|
11071
|
+
lastErrors = countErrors;
|
|
11072
|
+
validationAttempts++;
|
|
11073
|
+
continue;
|
|
11074
|
+
}
|
|
11075
|
+
}
|
|
9863
11076
|
break;
|
|
9864
11077
|
} catch (err) {
|
|
9865
11078
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -10835,11 +12048,21 @@ function parseSqlSeed(sql) {
|
|
|
10835
12048
|
function loadSeedStateFromPath(seedRoot, seedName) {
|
|
10836
12049
|
const jsonPath = resolve4(seedRoot, `${seedName}.json`);
|
|
10837
12050
|
if (existsSync10(jsonPath)) {
|
|
10838
|
-
|
|
12051
|
+
try {
|
|
12052
|
+
return JSON.parse(readFileSync12(jsonPath, "utf-8"));
|
|
12053
|
+
} catch (err) {
|
|
12054
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
12055
|
+
throw new Error(`Failed to parse seed file ${jsonPath}: ${detail}`);
|
|
12056
|
+
}
|
|
10839
12057
|
}
|
|
10840
12058
|
const sqlPath = resolve4(seedRoot, `${seedName}.sql`);
|
|
10841
12059
|
if (existsSync10(sqlPath)) {
|
|
10842
|
-
|
|
12060
|
+
try {
|
|
12061
|
+
return parseSqlSeed(readFileSync12(sqlPath, "utf-8"));
|
|
12062
|
+
} catch (err) {
|
|
12063
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
12064
|
+
throw new Error(`Failed to parse seed file ${sqlPath}: ${detail}`);
|
|
12065
|
+
}
|
|
10843
12066
|
}
|
|
10844
12067
|
return null;
|
|
10845
12068
|
}
|
|
@@ -11075,7 +12298,9 @@ ${baseTaskMessage}` : baseTaskMessage;
|
|
|
11075
12298
|
};
|
|
11076
12299
|
}
|
|
11077
12300
|
if (trace.length === 0) {
|
|
11078
|
-
warn(
|
|
12301
|
+
warn(
|
|
12302
|
+
`Agent made no tool calls on run ${runIndex + 1}. This usually means the model is too weak for this scenario. Try a more capable model (e.g. --engine-model claude-sonnet-4-6 or --engine-model gemini-2.5-pro). If using a custom agent, check that it correctly processes tool schemas and calls tools.`
|
|
12303
|
+
);
|
|
11079
12304
|
}
|
|
11080
12305
|
progress(`Evaluating run ${runIndex + 1}...`);
|
|
11081
12306
|
const evaluationResult = await evaluateRun(
|
|
@@ -11412,8 +12637,14 @@ Pass --allow-ambiguous-seed to opt into best-effort generation.`;
|
|
|
11412
12637
|
for (const sel of seedSelections) {
|
|
11413
12638
|
const mismatches = verifySeedCounts(scenario.setup, sel.seedData);
|
|
11414
12639
|
if (mismatches.length === 0) continue;
|
|
12640
|
+
const significantMismatches = mismatches.filter((m) => {
|
|
12641
|
+
const delta = Math.abs(m.expected - m.actual);
|
|
12642
|
+
const ratio = m.expected > 0 ? delta / m.expected : delta;
|
|
12643
|
+
return delta > 5 || ratio > 0.5;
|
|
12644
|
+
});
|
|
12645
|
+
if (significantMismatches.length === 0) continue;
|
|
11415
12646
|
warn(
|
|
11416
|
-
`Seed count mismatch for ${sel.twinName}: ${
|
|
12647
|
+
`Seed count mismatch for ${sel.twinName}: ${significantMismatches.map((m) => `${m.subject}: expected ${m.expected}, got ${m.actual}`).join("; ")}`
|
|
11417
12648
|
);
|
|
11418
12649
|
}
|
|
11419
12650
|
const scenarioDir = dirname2(resolve4(options.scenarioPath));
|
|
@@ -11605,7 +12836,7 @@ Pass --allow-ambiguous-seed to opt into best-effort generation.`;
|
|
|
11605
12836
|
printHeader(scenario.title, seedSelections);
|
|
11606
12837
|
const evaluatorProvider = detectProvider(model);
|
|
11607
12838
|
const configProvider = detectProvider(config.model);
|
|
11608
|
-
const evaluatorApiKey = options.model && evaluatorProvider !== configProvider ? resolveProviderApiKey("", evaluatorProvider) : resolveProviderApiKey(config.apiKey, evaluatorProvider);
|
|
12839
|
+
const evaluatorApiKey = config.evaluatorProvider === "archal" ? "" : options.model && evaluatorProvider !== configProvider ? resolveProviderApiKey("", evaluatorProvider) : resolveProviderApiKey(config.apiKey, evaluatorProvider);
|
|
11609
12840
|
const evaluatorConfig = {
|
|
11610
12841
|
apiKey: evaluatorApiKey,
|
|
11611
12842
|
model,
|
|
@@ -12132,7 +13363,10 @@ function createRunCommand() {
|
|
|
12132
13363
|
).addOption(new Option("--openclaw-url <url>", "Deprecated alias for --engine-endpoint").hideHelp()).addOption(new Option("--openclaw-token <token>", "Deprecated alias for --engine-token").hideHelp()).addOption(new Option("--openclaw-agent <id>", "Deprecated alias for --engine-model").hideHelp()).addOption(new Option("--openclaw-twin-urls <path>", "Deprecated alias for --engine-twin-urls").hideHelp()).addOption(new Option("--openclaw-timeout <seconds>", "Deprecated alias for --engine-timeout").hideHelp()).option("--api-base-urls <path>", "Path to JSON mapping service names to clone API base URLs for raw API code routing").option("--api-proxy-url <url>", "Proxy URL for raw API code routing metadata").option("--preflight-only", "Run environment/config preflight checks only and exit").option("--seed-cache", "Enable seed cache for dynamic generation (off by default)").option("--static-seed", "Use seed files as-is without LLM mutation (uses --seed name or auto-selected per twin)").option("--no-failure-analysis", "Skip LLM failure analysis on imperfect scores").option(
|
|
12133
13364
|
"--allow-ambiguous-seed",
|
|
12134
13365
|
"Allow dynamic seed generation when setup is underspecified"
|
|
12135
|
-
).option("--tag <tag>", "Only run if scenario has this tag (exit 0 if not)").option("-q, --quiet", "Suppress non-error output").option("-v, --verbose", "Enable debug logging").action(async (scenarioArg, opts) => {
|
|
13366
|
+
).option("--tag <tag>", "Only run if scenario has this tag (exit 0 if not)").option("-q, --quiet", "Suppress non-error output").option("-v, --verbose", "Enable debug logging").action(async (scenarioArg, opts, command) => {
|
|
13367
|
+
const parentOpts = command.parent?.opts() ?? {};
|
|
13368
|
+
if (parentOpts.quiet) opts.quiet = true;
|
|
13369
|
+
if (parentOpts.verbose) opts.verbose = true;
|
|
12136
13370
|
if (opts.quiet) {
|
|
12137
13371
|
configureLogger({ quiet: true });
|
|
12138
13372
|
}
|
|
@@ -12405,9 +13639,16 @@ function createRunCommand() {
|
|
|
12405
13639
|
}
|
|
12406
13640
|
if (opts.apiKey?.trim()) {
|
|
12407
13641
|
warnIfKeyLooksInvalid(opts.apiKey.trim(), "--api-key");
|
|
12408
|
-
|
|
13642
|
+
const key = opts.apiKey.trim();
|
|
13643
|
+
process.env["ARCHAL_ENGINE_API_KEY"] = key;
|
|
13644
|
+
if (key.startsWith("AIza")) {
|
|
13645
|
+
process.env["GEMINI_API_KEY"] = process.env["GEMINI_API_KEY"] || key;
|
|
13646
|
+
} else if (key.startsWith("sk-ant-")) {
|
|
13647
|
+
process.env["ANTHROPIC_API_KEY"] = process.env["ANTHROPIC_API_KEY"] || key;
|
|
13648
|
+
} else if (key.startsWith("sk-")) {
|
|
13649
|
+
process.env["OPENAI_API_KEY"] = process.env["OPENAI_API_KEY"] || key;
|
|
13650
|
+
}
|
|
12409
13651
|
if (!opts.engineModel && !process.env["ARCHAL_ENGINE_MODEL"] && !opts.model?.trim()) {
|
|
12410
|
-
const key = opts.apiKey.trim();
|
|
12411
13652
|
if (key.startsWith("AIza")) {
|
|
12412
13653
|
opts.engineModel = "gemini-2.0-flash";
|
|
12413
13654
|
} else if (key.startsWith("sk-ant-")) {
|
|
@@ -12578,13 +13819,13 @@ function createRunCommand() {
|
|
|
12578
13819
|
let statusReadySinceMs = null;
|
|
12579
13820
|
const isRetryablePollFailure = (result) => result.offline || typeof result.status === "number" && result.status >= 500;
|
|
12580
13821
|
const sleepForPollInterval = async () => new Promise((resolve12) => setTimeout(resolve12, SESSION_POLL_INTERVAL_MS));
|
|
12581
|
-
process.stderr.write("Starting cloud session...\n");
|
|
13822
|
+
if (!opts.quiet) process.stderr.write("Starting cloud session...\n");
|
|
12582
13823
|
let pollCount = 0;
|
|
12583
13824
|
while (Date.now() < readyDeadline) {
|
|
12584
13825
|
pollCount++;
|
|
12585
13826
|
if (pollCount % 4 === 0) {
|
|
12586
13827
|
const elapsedSec = Math.round((Date.now() - (readyDeadline - SESSION_READY_TIMEOUT_MS)) / 1e3);
|
|
12587
|
-
process.stderr.write(` Still waiting for session to be ready (${elapsedSec}s)...
|
|
13828
|
+
if (!opts.quiet) process.stderr.write(` Still waiting for session to be ready (${elapsedSec}s)...
|
|
12588
13829
|
`);
|
|
12589
13830
|
}
|
|
12590
13831
|
const freshCreds = getCredentials();
|
|
@@ -12655,7 +13896,7 @@ function createRunCommand() {
|
|
|
12655
13896
|
}
|
|
12656
13897
|
if (sessionReady) {
|
|
12657
13898
|
const warmupSec = Math.round((Date.now() - (readyDeadline - SESSION_READY_TIMEOUT_MS)) / 1e3);
|
|
12658
|
-
process.stderr.write(`Cloud session ready (${warmupSec}s).
|
|
13899
|
+
if (!opts.quiet) process.stderr.write(`Cloud session ready (${warmupSec}s).
|
|
12659
13900
|
`);
|
|
12660
13901
|
}
|
|
12661
13902
|
if (!sessionReady && !runFailureMessage) {
|
|
@@ -13018,12 +14259,33 @@ function buildEvidenceArtifacts(report) {
|
|
|
13018
14259
|
runIndex: run.runIndex,
|
|
13019
14260
|
steps: buildAgentTraceSteps(run)
|
|
13020
14261
|
})).filter((run) => run.steps.length > 0);
|
|
14262
|
+
const evaluations = reportRuns.flatMap(
|
|
14263
|
+
(run) => (run.evaluations ?? []).map((ev) => ({
|
|
14264
|
+
runIndex: run.runIndex,
|
|
14265
|
+
criterionId: ev.criterionId,
|
|
14266
|
+
status: ev.status,
|
|
14267
|
+
result: ev.status,
|
|
14268
|
+
confidence: ev.confidence,
|
|
14269
|
+
explanation: ev.explanation
|
|
14270
|
+
}))
|
|
14271
|
+
);
|
|
14272
|
+
const latestEvaluationByCriterion = /* @__PURE__ */ new Map();
|
|
14273
|
+
for (const evaluation of evaluations) {
|
|
14274
|
+
const current = latestEvaluationByCriterion.get(evaluation.criterionId);
|
|
14275
|
+
if (!current || evaluation.runIndex >= current.runIndex) {
|
|
14276
|
+
latestEvaluationByCriterion.set(evaluation.criterionId, evaluation);
|
|
14277
|
+
}
|
|
14278
|
+
}
|
|
13021
14279
|
const criteria = Object.entries(report.criterionDescriptions ?? {}).map(
|
|
13022
|
-
([id, description]) =>
|
|
13023
|
-
id
|
|
13024
|
-
|
|
13025
|
-
|
|
13026
|
-
|
|
14280
|
+
([id, description]) => {
|
|
14281
|
+
const latest = latestEvaluationByCriterion.get(id);
|
|
14282
|
+
return {
|
|
14283
|
+
id,
|
|
14284
|
+
label: description,
|
|
14285
|
+
kind: report.criterionTypes?.[id] ?? null,
|
|
14286
|
+
result: latest?.result ?? null
|
|
14287
|
+
};
|
|
14288
|
+
}
|
|
13027
14289
|
);
|
|
13028
14290
|
const runs = reportRuns.map((run) => ({
|
|
13029
14291
|
runIndex: run.runIndex,
|
|
@@ -13033,6 +14295,7 @@ function buildEvidenceArtifacts(report) {
|
|
|
13033
14295
|
evaluations: (run.evaluations ?? []).map((ev) => ({
|
|
13034
14296
|
criterionId: ev.criterionId,
|
|
13035
14297
|
status: ev.status,
|
|
14298
|
+
result: ev.status,
|
|
13036
14299
|
confidence: ev.confidence,
|
|
13037
14300
|
explanation: ev.explanation
|
|
13038
14301
|
}))
|
|
@@ -13041,6 +14304,7 @@ function buildEvidenceArtifacts(report) {
|
|
|
13041
14304
|
satisfaction: report.satisfactionScore,
|
|
13042
14305
|
scores: reportRuns.map((r) => r.overallScore),
|
|
13043
14306
|
criteria,
|
|
14307
|
+
evaluations,
|
|
13044
14308
|
runs,
|
|
13045
14309
|
traceEntries,
|
|
13046
14310
|
thinkingTraceEntries,
|
|
@@ -15118,9 +16382,10 @@ function findBundledScenarios() {
|
|
|
15118
16382
|
return results;
|
|
15119
16383
|
}
|
|
15120
16384
|
function detectProviderName(model) {
|
|
15121
|
-
|
|
15122
|
-
if (
|
|
15123
|
-
if (
|
|
16385
|
+
const normalized = model.toLowerCase();
|
|
16386
|
+
if (normalized.startsWith("gemini-")) return "Gemini";
|
|
16387
|
+
if (normalized.startsWith("claude-") || normalized.startsWith("sonnet-") || normalized.startsWith("haiku-") || normalized.startsWith("opus-")) return "Anthropic";
|
|
16388
|
+
if (normalized.startsWith("gpt-") || normalized.startsWith("o1-") || normalized.startsWith("o3-") || normalized.startsWith("o4-")) return "OpenAI";
|
|
15124
16389
|
return "OpenAI-compatible";
|
|
15125
16390
|
}
|
|
15126
16391
|
function resolveEngineApiKey(explicitKey) {
|
|
@@ -15189,6 +16454,13 @@ Set one via:
|
|
|
15189
16454
|
process.exit(1);
|
|
15190
16455
|
}
|
|
15191
16456
|
process.env["ARCHAL_ENGINE_API_KEY"] = engineApiKey;
|
|
16457
|
+
if (engineApiKey.startsWith("AIza")) {
|
|
16458
|
+
process.env["GEMINI_API_KEY"] = process.env["GEMINI_API_KEY"] || engineApiKey;
|
|
16459
|
+
} else if (engineApiKey.startsWith("sk-ant-")) {
|
|
16460
|
+
process.env["ANTHROPIC_API_KEY"] = process.env["ANTHROPIC_API_KEY"] || engineApiKey;
|
|
16461
|
+
} else if (engineApiKey.startsWith("sk-")) {
|
|
16462
|
+
process.env["OPENAI_API_KEY"] = process.env["OPENAI_API_KEY"] || engineApiKey;
|
|
16463
|
+
}
|
|
15192
16464
|
const runs = parseInt(opts.runs, 10);
|
|
15193
16465
|
if (Number.isNaN(runs) || runs <= 0) {
|
|
15194
16466
|
process.stderr.write("Error: --runs must be a positive integer\n");
|