@ak--47/dungeon-master 1.6.5 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -11
- package/.claude/skills/create-dungeon/SKILL.md +49 -10
- package/.claude/skills/create-project/SKILL.md +22 -3
- package/.claude/skills/create-project/context.mjs +89 -0
- package/.claude/skills/create-project/provision.mjs +1 -60
- package/.claude/skills/headless-build/SKILL.md +18 -1
- package/.claude/skills/powertools/SKILL.md +20 -1
- package/.claude/skills/release-check/SKILL.md +99 -0
- package/.claude/skills/verify-dungeon/SKILL.md +71 -16
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +14 -0
- package/.claude/skills/verify-dungeon/references/report-format.md +18 -1
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +36 -1
- package/.claude/skills/warehouse-metrics/GAPS-template.md +34 -0
- package/.claude/skills/warehouse-metrics/SKILL.md +105 -0
- package/.claude/skills/warehouse-metrics/deploy.mjs +651 -0
- package/.claude/skills/write-hooks/SKILL.md +33 -3
- package/CHANGELOG.md +331 -0
- package/HOOKS.md +154 -5
- package/README.md +357 -8
- package/docs/guides/1.7.0-upgrade-guide.md +154 -0
- package/docs/guides/1.8.0-upgrade-guide.md +151 -0
- package/dungeons/technical/warehouse.js +187 -0
- package/index.js +131 -2
- package/lib/core/config-validator.js +264 -13
- package/lib/core/context.js +39 -0
- package/lib/core/dungeon-loader.js +5 -2
- package/lib/core/storage.js +51 -3
- package/lib/generators/events.js +53 -7
- package/lib/generators/funnels.js +85 -11
- package/lib/generators/profiles.js +9 -4
- package/lib/generators/standalone.js +248 -0
- package/lib/generators/warehouse.js +828 -0
- package/lib/orchestrators/mixpanel-sender.js +39 -3
- package/lib/orchestrators/user-loop.js +240 -9
- package/lib/templates/story-spec.schema.json +41 -16
- package/lib/utils/conditions.js +62 -0
- package/lib/utils/json-evaluator.js +12 -2
- package/lib/utils/utils.js +115 -19
- package/lib/verify/index.js +1 -0
- package/lib/verify/schema-validator.js +8 -0
- package/lib/verify/story-runner.js +71 -8
- package/lib/verify/warehouse.js +683 -0
- package/package.json +5 -11
- package/scripts/verify-stories.mjs +150 -44
- package/types.d.ts +606 -38
|
@@ -15,6 +15,47 @@ import { makeName } from "ak-tools";
|
|
|
15
15
|
import * as u from "../utils/utils.js";
|
|
16
16
|
import { resolveSoup } from "../templates/soup-presets.js";
|
|
17
17
|
import { resolveMacro } from "../templates/macro-presets.js";
|
|
18
|
+
import { locations as LOCATION_TEMPLATE } from "../templates/defaults.js";
|
|
19
|
+
import { CONDITION_OPERATORS } from "../utils/conditions.js";
|
|
20
|
+
import { validateStandaloneEvents } from "../generators/standalone.js";
|
|
21
|
+
import { validateWarehouseMetrics } from "../generators/warehouse.js";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* v1.7.0 (P2-2): one entry per value the validator changed or flagged. Collected
|
|
25
|
+
* on `validatedConfig._warnings` and surfaced as `result.warnings` regardless of
|
|
26
|
+
* `verbose`. Console output stays `verbose`-gated; a config UI needs the data,
|
|
27
|
+
* not the log line.
|
|
28
|
+
*
|
|
29
|
+
* @typedef {import('../../types.js').EngineWarning} EngineWarning
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Resolve `singleCountry` to the canonical country name used by the location
|
|
34
|
+
* template. Accepts the full name (`"United States"`) or the ISO code (`"US"`),
|
|
35
|
+
* case-insensitive. Throws when nothing matches — an empty location pool
|
|
36
|
+
* silently deleted every geo property from events and profiles before 1.7.0.
|
|
37
|
+
*
|
|
38
|
+
* @param {unknown} singleCountry
|
|
39
|
+
* @returns {string | undefined} canonical `country` value, or undefined when unset
|
|
40
|
+
*/
|
|
41
|
+
export function resolveSingleCountry(singleCountry) {
|
|
42
|
+
if (singleCountry === undefined || singleCountry === null || singleCountry === '') return undefined;
|
|
43
|
+
if (typeof singleCountry !== 'string') {
|
|
44
|
+
throw new Error(`singleCountry must be a country name or ISO code string (got ${typeof singleCountry})`);
|
|
45
|
+
}
|
|
46
|
+
const needle = singleCountry.trim().toLowerCase();
|
|
47
|
+
const hit = LOCATION_TEMPLATE.find(l =>
|
|
48
|
+
String(l.country).toLowerCase() === needle || String(l.country_code).toLowerCase() === needle
|
|
49
|
+
);
|
|
50
|
+
if (!hit) {
|
|
51
|
+
const valid = [...new Map(LOCATION_TEMPLATE.map(l => [l.country_code, `${l.country_code} (${l.country})`])).values()];
|
|
52
|
+
throw new Error(
|
|
53
|
+
`singleCountry "${singleCountry}" matches no country in the location template, so the location pool would be empty ` +
|
|
54
|
+
`and every geo property would vanish. Valid values (name or code): ${valid.join(', ')}`
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
return String(hit.country);
|
|
58
|
+
}
|
|
18
59
|
|
|
19
60
|
/**
|
|
20
61
|
* Resolve dataset window from config. Returns { datasetStartUnix, datasetEndUnix, numDays }.
|
|
@@ -185,7 +226,9 @@ const CONFIG_SUBOBJECTS = {
|
|
|
185
226
|
// settable — line ~942 unconditionally overwrites it with
|
|
186
227
|
// `validatedEvents.some(e => e.isAttributionEvent)`. Listing it here presented
|
|
187
228
|
// a knob that never did anything.
|
|
188
|
-
|
|
229
|
+
// v1.7.0: `singleCountry`, `campaignPerUser` and `stickyEventProps` are data-shape
|
|
230
|
+
// knobs, so they hoist from `switches` like the booleans do.
|
|
231
|
+
switches: ['hasLocation', 'hasCampaigns', 'hasAdSpend', 'hasSessionIds', 'hasAvatar', 'hasIOSDevices', 'hasAndroidDevices', 'hasDesktopDevices', 'hasBrowser', 'isAnonymous', 'alsoInferFunnels', 'singleCountry', 'campaignPerUser', 'stickyEventProps'],
|
|
189
232
|
identity: ['avgDevicePerUser', 'sessionTimeout'],
|
|
190
233
|
};
|
|
191
234
|
|
|
@@ -262,7 +305,7 @@ function stripKilledConfigKeys(config) {
|
|
|
262
305
|
*
|
|
263
306
|
* @param {import('../../types.js').Funnel[]} funnels
|
|
264
307
|
*/
|
|
265
|
-
function validateConversionWindow(funnels, verbose = false) {
|
|
308
|
+
function validateConversionWindow(funnels, verbose = false, warnings = []) {
|
|
266
309
|
const DEFAULT_WINDOW_DAYS = 30;
|
|
267
310
|
const MAX_WINDOW_DAYS = 180;
|
|
268
311
|
for (const f of funnels) {
|
|
@@ -272,6 +315,13 @@ function validateConversionWindow(funnels, verbose = false) {
|
|
|
272
315
|
if (f.conversionWindowDays === undefined || f.conversionWindowDays === null) {
|
|
273
316
|
if (ttcDays >= DEFAULT_WINDOW_DAYS) {
|
|
274
317
|
f.conversionWindowDays = Math.min(MAX_WINDOW_DAYS, Math.ceil(ttcDays * 1.5));
|
|
318
|
+
warnings.push({
|
|
319
|
+
key: `funnels[${f.name || (f.sequence && f.sequence.join(' > '))}].conversionWindowDays`,
|
|
320
|
+
requested: undefined,
|
|
321
|
+
applied: f.conversionWindowDays,
|
|
322
|
+
reason: `timeToConvert (${ttcDays.toFixed(1)}d) exceeds the default 30d conversion window; auto-set`,
|
|
323
|
+
severity: 'warn',
|
|
324
|
+
});
|
|
275
325
|
if (verbose) console.warn(
|
|
276
326
|
`⚠️ Funnel "${f.name || (f.sequence && f.sequence.join(' > '))}": ` +
|
|
277
327
|
`timeToConvert (${ttcDays.toFixed(1)}d) exceeds default 30d conversion window. ` +
|
|
@@ -385,8 +435,112 @@ function normalizeExperiments(funnels, datasetEndUnix) {
|
|
|
385
435
|
// Sticky bucketing defaults true — the per-user hash was the only pre-1.6
|
|
386
436
|
// behavior, so existing dungeons stay byte-identical.
|
|
387
437
|
const sticky = raw.sticky === undefined ? true : !!raw.sticky;
|
|
388
|
-
|
|
438
|
+
// v1.7.0 (P0-2): stamp the assigned variant on the user profile as
|
|
439
|
+
// `Experiment: <name>`. Default true; `stampProfile: false` opts out.
|
|
440
|
+
// Only meaningful when sticky — a re-rolled variant has no single value.
|
|
441
|
+
const stampProfile = raw.stampProfile === undefined ? true : !!raw.stampProfile;
|
|
442
|
+
f._experiment = { name, variants, startUnix, sticky, stampProfile };
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* v1.7.0 (P0-1): validate `Funnel.conditions` shapes at config time.
|
|
448
|
+
*
|
|
449
|
+
* Throws on shapes that can never match: function values, bare arrays (use
|
|
450
|
+
* `{ in: [...] }`), unknown operators, `in`/`nin` without an array, ordering
|
|
451
|
+
* operators without a number/string. Warns (into `warnings`) when a condition
|
|
452
|
+
* key is declared nowhere the profile is built from — `userProps`, any persona's
|
|
453
|
+
* `properties`, or `superProps` — because only a `user` hook could then supply it.
|
|
454
|
+
*
|
|
455
|
+
* @param {import('../../types.js').Funnel[]} funnels
|
|
456
|
+
* @param {Partial<Dungeon>} config
|
|
457
|
+
* @param {EngineWarning[]} warnings
|
|
458
|
+
*/
|
|
459
|
+
function validateFunnelConditions(funnels, config, warnings) {
|
|
460
|
+
if (!Array.isArray(funnels)) return;
|
|
461
|
+
const declared = new Set([
|
|
462
|
+
...Object.keys(config.userProps || {}),
|
|
463
|
+
...Object.keys(config.superProps || {}),
|
|
464
|
+
...(Array.isArray(config.personas) ? config.personas.flatMap(p => Object.keys((p && p.properties) || {})) : []),
|
|
465
|
+
]);
|
|
466
|
+
const undeclared = new Set();
|
|
467
|
+
for (const f of funnels) {
|
|
468
|
+
if (!f || !f.conditions) continue;
|
|
469
|
+
const label = `funnels[${f.name || (Array.isArray(f.sequence) ? f.sequence.join(' > ') : '?')}].conditions`;
|
|
470
|
+
if (typeof f.conditions !== 'object' || Array.isArray(f.conditions)) {
|
|
471
|
+
throw new Error(`${label} must be an object mapping profile keys to a scalar or an operator map`);
|
|
472
|
+
}
|
|
473
|
+
for (const [key, cond] of Object.entries(f.conditions)) {
|
|
474
|
+
if (typeof cond === 'function') {
|
|
475
|
+
throw new Error(`${label}.${key} is a function. Conditions compare against the resolved profile value; pass a scalar or an operator map such as { in: [...] }.`);
|
|
476
|
+
}
|
|
477
|
+
if (Array.isArray(cond)) {
|
|
478
|
+
throw new Error(`${label}.${key} is a bare array, which never matches (strict equality). Use { in: [${cond.map(v => JSON.stringify(v)).join(', ')}] }.`);
|
|
479
|
+
}
|
|
480
|
+
if (cond !== null && typeof cond === 'object' && !(cond instanceof Date)) {
|
|
481
|
+
const ops = Object.keys(cond);
|
|
482
|
+
if (!ops.length) throw new Error(`${label}.${key} is an empty operator map`);
|
|
483
|
+
for (const op of ops) {
|
|
484
|
+
if (!CONDITION_OPERATORS.includes(op)) {
|
|
485
|
+
throw new Error(`${label}.${key} uses unknown operator "${op}". Valid operators: ${CONDITION_OPERATORS.join(', ')}`);
|
|
486
|
+
}
|
|
487
|
+
const expected = cond[op];
|
|
488
|
+
if ((op === 'in' || op === 'nin') && !Array.isArray(expected)) {
|
|
489
|
+
throw new Error(`${label}.${key}.${op} requires an array (got ${typeof expected})`);
|
|
490
|
+
}
|
|
491
|
+
if ((op === 'gt' || op === 'gte' || op === 'lt' || op === 'lte') && typeof expected !== 'number' && typeof expected !== 'string') {
|
|
492
|
+
throw new Error(`${label}.${key}.${op} requires a number or string (got ${typeof expected})`);
|
|
493
|
+
}
|
|
494
|
+
if ((op === 'eq' || op === 'neq') && (typeof expected === 'function' || Array.isArray(expected))) {
|
|
495
|
+
throw new Error(`${label}.${key}.${op} requires a scalar`);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
if (!declared.has(key)) undeclared.add(`${label}.${key}`);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
for (const path of undeclared) {
|
|
503
|
+
warnings.push({
|
|
504
|
+
key: path,
|
|
505
|
+
requested: undefined,
|
|
506
|
+
applied: undefined,
|
|
507
|
+
reason: 'condition key is not declared in userProps, superProps, or any persona.properties — it can only match if a `user` hook sets it',
|
|
508
|
+
severity: 'warn',
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/**
|
|
514
|
+
* v1.7.0 (P1-2): validate `stickyEventProps`. Every key must be declared in
|
|
515
|
+
* `userProps`, a persona's `properties`, or `superProps` (schema-first). Returns
|
|
516
|
+
* the normalized list plus the subset that lives only in `superProps` (those are
|
|
517
|
+
* resolved once per user rather than copied from the profile).
|
|
518
|
+
*
|
|
519
|
+
* @param {unknown} stickyEventProps
|
|
520
|
+
* @param {Partial<Dungeon>} config
|
|
521
|
+
* @returns {{ keys: string[], superOnly: Set<string> }}
|
|
522
|
+
*/
|
|
523
|
+
function validateStickyEventProps(stickyEventProps, config) {
|
|
524
|
+
if (stickyEventProps === undefined || stickyEventProps === null) return { keys: [], superOnly: new Set() };
|
|
525
|
+
if (!Array.isArray(stickyEventProps) || stickyEventProps.some(k => typeof k !== 'string' || !k)) {
|
|
526
|
+
throw new Error(`stickyEventProps must be an array of property-name strings (got ${JSON.stringify(stickyEventProps)})`);
|
|
389
527
|
}
|
|
528
|
+
const profileKeys = new Set([
|
|
529
|
+
...Object.keys(config.userProps || {}),
|
|
530
|
+
...(Array.isArray(config.personas) ? config.personas.flatMap(p => Object.keys((p && p.properties) || {})) : []),
|
|
531
|
+
]);
|
|
532
|
+
const superKeys = new Set(Object.keys(config.superProps || {}));
|
|
533
|
+
const superOnly = new Set();
|
|
534
|
+
const keys = [...new Set(stickyEventProps)];
|
|
535
|
+
for (const key of keys) {
|
|
536
|
+
if (profileKeys.has(key)) continue;
|
|
537
|
+
if (superKeys.has(key)) { superOnly.add(key); continue; }
|
|
538
|
+
throw new Error(
|
|
539
|
+
`stickyEventProps entry "${key}" is not declared in userProps, superProps, or any persona.properties. ` +
|
|
540
|
+
`Declare it first (schema-first) — hooks may not add new properties.`
|
|
541
|
+
);
|
|
542
|
+
}
|
|
543
|
+
return { keys, superOnly };
|
|
390
544
|
}
|
|
391
545
|
|
|
392
546
|
/**
|
|
@@ -493,9 +647,51 @@ export function validateDungeonConfig(config) {
|
|
|
493
647
|
name = "",
|
|
494
648
|
batchSize = 2_500_000,
|
|
495
649
|
concurrency = 1,
|
|
496
|
-
strictEventCount = false
|
|
650
|
+
strictEventCount = false,
|
|
651
|
+
autoPowerLaw = true,
|
|
652
|
+
campaignPerUser = false,
|
|
497
653
|
} = config;
|
|
498
654
|
|
|
655
|
+
/** @type {EngineWarning[]} */
|
|
656
|
+
const warnings = [];
|
|
657
|
+
|
|
658
|
+
// v1.8.0 — identity-less metric snapshots. Throws on anything malformed;
|
|
659
|
+
// a silent skip would drop a whole data stream without the author noticing.
|
|
660
|
+
const standaloneEvents = validateStandaloneEvents(config.standaloneEvents);
|
|
661
|
+
const {
|
|
662
|
+
warehouseMetrics,
|
|
663
|
+
warnings: warehouseMetricWarnings,
|
|
664
|
+
} = validateWarehouseMetrics(config);
|
|
665
|
+
for (const reason of warehouseMetricWarnings) {
|
|
666
|
+
const key = String(reason).split(' ')[0] || 'warehouseMetrics';
|
|
667
|
+
warnings.push({
|
|
668
|
+
key,
|
|
669
|
+
reason,
|
|
670
|
+
severity: /clamp/i.test(reason) ? 'clamp' : 'warn',
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
// v1.7.0 (R2-2): resolve singleCountry to the template's canonical name (accepts
|
|
675
|
+
// ISO code or full name, case-insensitive) or throw. Before 1.7.0 a miss
|
|
676
|
+
// filtered the location pool to empty and silently deleted every geo property.
|
|
677
|
+
const singleCountry = resolveSingleCountry(config.singleCountry);
|
|
678
|
+
|
|
679
|
+
if (typeof autoPowerLaw !== 'boolean') {
|
|
680
|
+
throw new Error(`autoPowerLaw must be a boolean (got ${typeof autoPowerLaw})`);
|
|
681
|
+
}
|
|
682
|
+
if (typeof campaignPerUser !== 'boolean') {
|
|
683
|
+
throw new Error(`campaignPerUser must be a boolean (got ${typeof campaignPerUser})`);
|
|
684
|
+
}
|
|
685
|
+
if (campaignPerUser && !hasCampaigns) {
|
|
686
|
+
warnings.push({
|
|
687
|
+
key: 'campaignPerUser',
|
|
688
|
+
requested: true,
|
|
689
|
+
applied: false,
|
|
690
|
+
reason: 'campaignPerUser has no effect without hasCampaigns: true (no UTMs are stamped)',
|
|
691
|
+
severity: 'warn',
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
|
|
499
695
|
// Allow concurrency override from config (default is now 1)
|
|
500
696
|
if (config.concurrency === undefined || config.concurrency === null) {
|
|
501
697
|
concurrency = 1;
|
|
@@ -662,10 +858,12 @@ export function validateDungeonConfig(config) {
|
|
|
662
858
|
}
|
|
663
859
|
if (percentUsersBornInDataset > 100) {
|
|
664
860
|
if (verbose) console.warn(`⚠️ percentUsersBornInDataset=${percentUsersBornInDataset} clamped to 100. Values above 100 are not meaningful.`);
|
|
861
|
+
warnings.push({ key: 'percentUsersBornInDataset', requested: percentUsersBornInDataset, applied: 100, reason: 'values above 100 are not meaningful', severity: 'clamp' });
|
|
665
862
|
percentUsersBornInDataset = 100;
|
|
666
863
|
}
|
|
667
864
|
if (percentUsersBornInDataset < 0) {
|
|
668
865
|
if (verbose) console.warn(`⚠️ percentUsersBornInDataset=${percentUsersBornInDataset} clamped to 0. Negative values are not meaningful.`);
|
|
866
|
+
warnings.push({ key: 'percentUsersBornInDataset', requested: percentUsersBornInDataset, applied: 0, reason: 'negative values are not meaningful', severity: 'clamp' });
|
|
669
867
|
percentUsersBornInDataset = 0;
|
|
670
868
|
}
|
|
671
869
|
|
|
@@ -679,20 +877,32 @@ export function validateDungeonConfig(config) {
|
|
|
679
877
|
// dungeons that set percentUsersBornInDataset directly without picking a
|
|
680
878
|
// macro keep their existing behavior. Tuned empirically against the
|
|
681
879
|
// engine-validation sweep matrix (research/engine-sweep-pass*.json).
|
|
880
|
+
//
|
|
881
|
+
// v1.7.0 (R2-1): the cap is keyed on the NAMED preset only. A macro object
|
|
882
|
+
// without `preset` (`macro: { bornRecentBias: 0.3, percentUsersBornInDataset: 50 }`)
|
|
883
|
+
// is a custom macro — the author owns the shape, so no preset contract applies
|
|
884
|
+
// and the overrides are honored as written. Before 1.7.0 the preset-less object
|
|
885
|
+
// fell back to the `flat` cap (12) and silently ignored its own born%.
|
|
682
886
|
const MACRO_BORN_CAP = { flat: 12, steady: 12, growth: 30, viral: 55, decline: 5 };
|
|
683
|
-
const macroExplicit = config.macro !== undefined && config.macro !== null;
|
|
684
887
|
const macroKey = (typeof config.macro === 'string')
|
|
685
888
|
? config.macro
|
|
686
|
-
: (
|
|
687
|
-
if (userBornExplicit &&
|
|
889
|
+
: (macroAsObj !== null && typeof macroAsObj.preset === 'string') ? macroAsObj.preset : null;
|
|
890
|
+
if (userBornExplicit && macroKey !== null && MACRO_BORN_CAP[macroKey] !== undefined) {
|
|
688
891
|
const cap = MACRO_BORN_CAP[macroKey];
|
|
689
892
|
if (percentUsersBornInDataset > cap) {
|
|
690
893
|
if (verbose) console.warn(
|
|
691
894
|
`⚠️ macro="${macroKey}" + percentUsersBornInDataset=${percentUsersBornInDataset} ` +
|
|
692
895
|
`clamped to ${cap}. High born% with macro="${macroKey}" produces cumulative-acquisition ` +
|
|
693
|
-
`right-edge explosion. Use macro="growth" or "viral" for genuinely high-born configs
|
|
694
|
-
`
|
|
896
|
+
`right-edge explosion. Use macro="growth" or "viral" for genuinely high-born configs, ` +
|
|
897
|
+
`or drop \`preset\` for a custom macro with no cap.`
|
|
695
898
|
);
|
|
899
|
+
warnings.push({
|
|
900
|
+
key: 'percentUsersBornInDataset',
|
|
901
|
+
requested: percentUsersBornInDataset,
|
|
902
|
+
applied: cap,
|
|
903
|
+
reason: `macro preset "${macroKey}" caps percentUsersBornInDataset at ${cap} to keep its shape; use a higher-born preset or a custom macro object without \`preset\``,
|
|
904
|
+
severity: 'clamp',
|
|
905
|
+
});
|
|
696
906
|
percentUsersBornInDataset = cap;
|
|
697
907
|
}
|
|
698
908
|
}
|
|
@@ -702,10 +912,12 @@ export function validateDungeonConfig(config) {
|
|
|
702
912
|
if (userBiasExplicit) {
|
|
703
913
|
if (bornRecentBias > 0.5) {
|
|
704
914
|
if (verbose) console.warn(`⚠️ bornRecentBias=${bornRecentBias} clamped to 0.5. Above 0.5 produces unusable right-skew. To suppress, fix the config.`);
|
|
915
|
+
warnings.push({ key: 'bornRecentBias', requested: bornRecentBias, applied: 0.5, reason: 'above 0.5 produces unusable right-skew', severity: 'clamp' });
|
|
705
916
|
bornRecentBias = 0.5;
|
|
706
917
|
}
|
|
707
918
|
if (bornRecentBias < -0.5) {
|
|
708
919
|
if (verbose) console.warn(`⚠️ bornRecentBias=${bornRecentBias} clamped to -0.5. Below -0.5 produces unusable left-skew. To suppress, fix the config.`);
|
|
920
|
+
warnings.push({ key: 'bornRecentBias', requested: bornRecentBias, applied: -0.5, reason: 'below -0.5 produces unusable left-skew', severity: 'clamp' });
|
|
709
921
|
bornRecentBias = -0.5;
|
|
710
922
|
}
|
|
711
923
|
}
|
|
@@ -717,6 +929,7 @@ export function validateDungeonConfig(config) {
|
|
|
717
929
|
`⚠️ percentUsersBornInDataset=${percentUsersBornInDataset} + bornRecentBias=${bornRecentBias} ` +
|
|
718
930
|
`compounds to right-edge explosion. Clamping bornRecentBias to 0.3. To suppress, fix the config.`
|
|
719
931
|
);
|
|
932
|
+
warnings.push({ key: 'bornRecentBias', requested: bornRecentBias, applied: 0.3, reason: `percentUsersBornInDataset=${percentUsersBornInDataset} with bornRecentBias > 0.4 compounds to a right-edge explosion`, severity: 'clamp' });
|
|
720
933
|
bornRecentBias = 0.3;
|
|
721
934
|
}
|
|
722
935
|
|
|
@@ -727,6 +940,7 @@ export function validateDungeonConfig(config) {
|
|
|
727
940
|
// load + memory cost. Plan PROMPT.md: clamp to 50.
|
|
728
941
|
if (Number.isFinite(avgEventsPerUserPerDay) && avgEventsPerUserPerDay > 50) {
|
|
729
942
|
if (verbose) console.warn(`⚠️ avgEventsPerUserPerDay=${avgEventsPerUserPerDay} clamped to 50. Above 50 is unrealistic load + memory cost. To suppress, fix the config.`);
|
|
943
|
+
warnings.push({ key: 'avgEventsPerUserPerDay', requested: avgEventsPerUserPerDay, applied: 50, reason: 'above 50 is unrealistic load and memory cost; numEvents recomputed from the clamped rate', severity: 'clamp' });
|
|
730
944
|
avgEventsPerUserPerDay = 50;
|
|
731
945
|
numEvents = Math.round(avgEventsPerUserPerDay * numUsers * numDays);
|
|
732
946
|
}
|
|
@@ -737,6 +951,7 @@ export function validateDungeonConfig(config) {
|
|
|
737
951
|
if (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && avgActiveDaysPerUser > numDays * 0.5) {
|
|
738
952
|
const cap = Math.max(1, Math.floor(numDays * 0.5));
|
|
739
953
|
if (verbose) console.warn(`⚠️ avgActiveDaysPerUser=${avgActiveDaysPerUser} > numDays/2 (${numDays}/2); clamped to ${cap}. Above 50% defeats the concentrator purpose. To suppress, fix the config.`);
|
|
954
|
+
warnings.push({ key: 'avgActiveDaysPerUser', requested: avgActiveDaysPerUser, applied: cap, reason: `above numDays/2 (${numDays}/2) defeats the concentrator purpose`, severity: 'clamp' });
|
|
740
955
|
avgActiveDaysClamped = cap;
|
|
741
956
|
}
|
|
742
957
|
|
|
@@ -745,8 +960,9 @@ export function validateDungeonConfig(config) {
|
|
|
745
960
|
// already been resolved upstream — clamping numDays alone would desync the
|
|
746
961
|
// engine. Pre-validator numDays bound is preferred (validator throws on
|
|
747
962
|
// numDays <= 0 already at line ~422). Just warn for visibility.
|
|
748
|
-
if (
|
|
749
|
-
console.warn(`⚠️ numDays=${numDays} < 14. Strict-bar engine-validation metrics use 14-day windows; results are noisy below 14 days. Consider increasing numDays.`);
|
|
963
|
+
if (numDays < 14) {
|
|
964
|
+
if (verbose) console.warn(`⚠️ numDays=${numDays} < 14. Strict-bar engine-validation metrics use 14-day windows; results are noisy below 14 days. Consider increasing numDays.`);
|
|
965
|
+
warnings.push({ key: 'numDays', requested: numDays, applied: numDays, reason: 'below the 14-day safe range; trend shape is noisy and not clamped', severity: 'warn' });
|
|
750
966
|
}
|
|
751
967
|
|
|
752
968
|
// v1.6.4 (P3-9): warn when `avgActiveDaysPerUser` and `engagementDecay` are both
|
|
@@ -764,6 +980,7 @@ export function validateDungeonConfig(config) {
|
|
|
764
980
|
`Decay erodes the effective active-day count, so the realized value will be at or below ` +
|
|
765
981
|
`${avgActiveDaysPerUser}, never above. Prefer one knob or the other. See HOOKS.md §2.5.`
|
|
766
982
|
);
|
|
983
|
+
warnings.push({ key: 'avgActiveDaysPerUser', requested: avgActiveDaysPerUser, applied: avgActiveDaysPerUser, reason: 'set together with engagementDecay; decay erodes the realized active-day count below this value', severity: 'warn' });
|
|
767
984
|
}
|
|
768
985
|
// ──────────────────────────────────────────────────────────────────────
|
|
769
986
|
|
|
@@ -887,6 +1104,9 @@ export function validateDungeonConfig(config) {
|
|
|
887
1104
|
// Explicit user-defined funnels keep their declared `timeToConvert`.
|
|
888
1105
|
timeToConvert: 24,
|
|
889
1106
|
requireRepeats: false,
|
|
1107
|
+
// v1.7.0: marks the engine-synthesized catch-all so the "user matched no
|
|
1108
|
+
// funnel" report (P0-1 item 5) counts only author-declared funnels.
|
|
1109
|
+
_catchAll: true,
|
|
890
1110
|
});
|
|
891
1111
|
}
|
|
892
1112
|
|
|
@@ -918,6 +1138,12 @@ export function validateDungeonConfig(config) {
|
|
|
918
1138
|
worldEvents = resolveWorldEvents(worldEvents, datasetStartUnix);
|
|
919
1139
|
}
|
|
920
1140
|
|
|
1141
|
+
// v1.7.0 (P0-1): reject condition shapes that can never match; warn on undeclared keys.
|
|
1142
|
+
validateFunnelConditions(funnels, config, warnings);
|
|
1143
|
+
|
|
1144
|
+
// v1.7.0 (P1-2): schema-first check on stickyEventProps.
|
|
1145
|
+
const sticky = validateStickyEventProps(config.stickyEventProps, config);
|
|
1146
|
+
|
|
921
1147
|
// Feature 3: Engagement Decay
|
|
922
1148
|
let engagementDecay = config.engagementDecay || null;
|
|
923
1149
|
if (engagementDecay) {
|
|
@@ -937,7 +1163,7 @@ export function validateDungeonConfig(config) {
|
|
|
937
1163
|
validateAttempts(funnels);
|
|
938
1164
|
|
|
939
1165
|
// v1.5: default + auto-bump Funnel.conversionWindowDays.
|
|
940
|
-
validateConversionWindow(funnels, verbose);
|
|
1166
|
+
validateConversionWindow(funnels, verbose, warnings);
|
|
941
1167
|
|
|
942
1168
|
// v1.5.0: validate Funnel.exclusionEvents — entries must exist in events[].
|
|
943
1169
|
validateExclusionEvents(funnels, validatedEvents, verbose);
|
|
@@ -998,6 +1224,9 @@ export function validateDungeonConfig(config) {
|
|
|
998
1224
|
groupKeys: normalizedGroupKeys,
|
|
999
1225
|
groupProps,
|
|
1000
1226
|
lookupTables,
|
|
1227
|
+
// v1.8.0 identity-less metric snapshots (normalized)
|
|
1228
|
+
standaloneEvents,
|
|
1229
|
+
warehouseMetrics,
|
|
1001
1230
|
hasAnonIds: hasAnonIdsResolved,
|
|
1002
1231
|
avgDevicePerUser,
|
|
1003
1232
|
hasSessionIds,
|
|
@@ -1021,6 +1250,15 @@ export function validateDungeonConfig(config) {
|
|
|
1021
1250
|
hasIOSDevices,
|
|
1022
1251
|
name,
|
|
1023
1252
|
strictEventCount,
|
|
1253
|
+
// v1.7.0 knobs
|
|
1254
|
+
singleCountry,
|
|
1255
|
+
autoPowerLaw,
|
|
1256
|
+
campaignPerUser,
|
|
1257
|
+
stickyEventProps: sticky.keys,
|
|
1258
|
+
/** @internal keys in stickyEventProps that live only in superProps (resolved once per user) */
|
|
1259
|
+
_stickySuperOnly: [...sticky.superOnly],
|
|
1260
|
+
/** @internal v1.7.0 (P2-2): validator clamps + warnings; surfaced as `result.warnings` */
|
|
1261
|
+
_warnings: warnings,
|
|
1024
1262
|
// Macro trend (resolved from preset + per-dungeon overrides; clamped)
|
|
1025
1263
|
macro: config.macro,
|
|
1026
1264
|
bornRecentBias,
|
|
@@ -1189,7 +1427,11 @@ function validatePersonas(personas) {
|
|
|
1189
1427
|
}
|
|
1190
1428
|
if (p.eventMultiplier === undefined) p.eventMultiplier = 1.0;
|
|
1191
1429
|
if (p.conversionModifier === undefined) p.conversionModifier = 1.0;
|
|
1192
|
-
|
|
1430
|
+
// v1.7.0 (P1-3): time-to-convert multiplier, symmetrical with conversionModifier.
|
|
1431
|
+
if (p.ttcModifier === undefined) p.ttcModifier = 1.0;
|
|
1432
|
+
if (!Number.isFinite(p.ttcModifier) || p.ttcModifier <= 0) {
|
|
1433
|
+
throw new Error(`Persona "${p.name}" ttcModifier must be a positive finite number (got ${p.ttcModifier})`);
|
|
1434
|
+
}
|
|
1193
1435
|
if (p.properties === undefined) p.properties = {};
|
|
1194
1436
|
}
|
|
1195
1437
|
if (deadFieldsSet.size > 0 && !warnedDeadPersonaFields) {
|
|
@@ -1223,6 +1465,15 @@ function resolveWorldEvents(worldEvents, beginUnix) {
|
|
|
1223
1465
|
if (!we.affectsEvents) resolved.affectsEvents = "*";
|
|
1224
1466
|
if (!we.volumeMultiplier) resolved.volumeMultiplier = 1.0;
|
|
1225
1467
|
if (!we.conversionModifier) resolved.conversionModifier = 1.0;
|
|
1468
|
+
// v1.7.0 (P0-3): amplification is now implemented, so the field must be a
|
|
1469
|
+
// usable number. Above-1 values clone affected events across the window.
|
|
1470
|
+
if (!Number.isFinite(resolved.volumeMultiplier) || resolved.volumeMultiplier < 0) {
|
|
1471
|
+
throw new Error(`worldEvents["${we.name}"].volumeMultiplier must be a non-negative finite number (got ${we.volumeMultiplier})`);
|
|
1472
|
+
}
|
|
1473
|
+
if (resolved.aftermath && resolved.aftermath.volumeMultiplier !== undefined &&
|
|
1474
|
+
(!Number.isFinite(resolved.aftermath.volumeMultiplier) || resolved.aftermath.volumeMultiplier < 0)) {
|
|
1475
|
+
throw new Error(`worldEvents["${we.name}"].aftermath.volumeMultiplier must be a non-negative finite number (got ${resolved.aftermath.volumeMultiplier})`);
|
|
1476
|
+
}
|
|
1226
1477
|
return resolved;
|
|
1227
1478
|
}).sort((a, b) => a.startUnix - b.startUnix || (a.name || '').localeCompare(b.name || ''));
|
|
1228
1479
|
}
|
package/lib/core/context.js
CHANGED
|
@@ -75,6 +75,10 @@ function createRuntimeState() {
|
|
|
75
75
|
eventCount: 0,
|
|
76
76
|
storedEventCount: 0,
|
|
77
77
|
userCount: 0,
|
|
78
|
+
// v1.7.0 (R2-5): profile receipt counters, incremented at push time so
|
|
79
|
+
// they are correct in batch mode too (the in-memory array is flushed).
|
|
80
|
+
profilesGenerated: 0,
|
|
81
|
+
profilesDropped: 0,
|
|
78
82
|
isBatchMode: false,
|
|
79
83
|
verbose: false
|
|
80
84
|
};
|
|
@@ -157,6 +161,12 @@ export function createContext(config, storage = null, timeConstants = {}) {
|
|
|
157
161
|
|
|
158
162
|
const { reportProgress, getProgressSummary } = createProgressReporter(config);
|
|
159
163
|
|
|
164
|
+
// v1.7.0 (P2-2 / P2-4): runtime warning sink. Generators aggregate by `key`
|
|
165
|
+
// (one entry per funnel / knob, with a `count`) — never one entry per user,
|
|
166
|
+
// because the callers sit in the per-user hot loop.
|
|
167
|
+
/** @type {Map<string, import('../../types.js').EngineWarning>} */
|
|
168
|
+
const runtimeWarnings = new Map();
|
|
169
|
+
|
|
160
170
|
const context = {
|
|
161
171
|
config,
|
|
162
172
|
storage,
|
|
@@ -166,6 +176,35 @@ export function createContext(config, storage = null, timeConstants = {}) {
|
|
|
166
176
|
reportProgress,
|
|
167
177
|
getProgressSummary,
|
|
168
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Record (or bump) an aggregated runtime warning.
|
|
181
|
+
* @param {import('../../types.js').EngineWarning} entry
|
|
182
|
+
*/
|
|
183
|
+
addWarning(entry) {
|
|
184
|
+
const existing = runtimeWarnings.get(entry.key);
|
|
185
|
+
if (existing) {
|
|
186
|
+
existing.count = (existing.count || 1) + 1;
|
|
187
|
+
if (typeof entry.requested === 'number' && typeof existing.requested === 'number') {
|
|
188
|
+
existing.requested = Math.max(existing.requested, entry.requested);
|
|
189
|
+
}
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
runtimeWarnings.set(entry.key, { ...entry, count: entry.count || 1 });
|
|
193
|
+
},
|
|
194
|
+
|
|
195
|
+
/** @returns {import('../../types.js').EngineWarning[]} */
|
|
196
|
+
getWarnings() {
|
|
197
|
+
return [...runtimeWarnings.values()];
|
|
198
|
+
},
|
|
199
|
+
|
|
200
|
+
incrementProfilesGenerated() {
|
|
201
|
+
runtime.profilesGenerated++;
|
|
202
|
+
},
|
|
203
|
+
|
|
204
|
+
incrementProfilesDropped() {
|
|
205
|
+
runtime.profilesDropped++;
|
|
206
|
+
},
|
|
207
|
+
|
|
169
208
|
// Helper methods for updating state
|
|
170
209
|
incrementOperations() {
|
|
171
210
|
runtime.operations++;
|
|
@@ -202,7 +202,10 @@ function reviveFunctionObject(obj) {
|
|
|
202
202
|
if (typeof fn === 'function') {
|
|
203
203
|
// Smoke-test: call it once to verify it doesn't reference other missing variables.
|
|
204
204
|
// If it throws (e.g., referencing `items` from a lost closure), discard it.
|
|
205
|
-
|
|
205
|
+
// v1.7.0 (P1-1): context-aware bodies (`(ctx) => ctx.profile.plan`) get an
|
|
206
|
+
// empty ValueContext so the probe does not throw on `ctx` itself.
|
|
207
|
+
const probeCtx = { profile: {}, event: {}, time: 0, config: {} };
|
|
208
|
+
try { fn.length >= 1 ? fn(probeCtx) : fn(); } catch { return null; }
|
|
206
209
|
return fn;
|
|
207
210
|
}
|
|
208
211
|
} catch {
|
|
@@ -275,7 +278,7 @@ export function validateDungeonShape(input) {
|
|
|
275
278
|
'events', 'numEvents', 'numUsers', 'numDays', 'funnels',
|
|
276
279
|
'userProps', 'superProps', 'hook', 'token', 'seed',
|
|
277
280
|
'scdProps', 'groupKeys', 'lookupTables', 'mirrorProps',
|
|
278
|
-
'hasAdSpend', 'soup', 'format', 'writeToDisk'
|
|
281
|
+
'hasAdSpend', 'standaloneEvents', 'warehouseMetrics', 'soup', 'format', 'writeToDisk'
|
|
279
282
|
];
|
|
280
283
|
|
|
281
284
|
const hasAnyDungeonKey = dungeonKeys.some(key => key in config);
|
package/lib/core/storage.js
CHANGED
|
@@ -91,15 +91,17 @@ export async function createHookArray(arr = [], opts) {
|
|
|
91
91
|
if (item === null || item === undefined) return false;
|
|
92
92
|
if (typeof item === 'object' && Object.keys(item).length === 0) return false;
|
|
93
93
|
|
|
94
|
+
const isWarehouse = type === "warehouse";
|
|
95
|
+
|
|
94
96
|
// Skip hook for types already hooked in generators/orchestrators to prevent double-firing
|
|
95
97
|
// Types hooked upstream: "event" (events.js), "user" (user-loop.js), "scd" (user-loop.js)
|
|
96
|
-
// Types only hooked here: "mirror", "ad-spend", "group", "lookup"
|
|
98
|
+
// Types only hooked here: "mirror", "ad-spend", "group", "lookup", "standalone"
|
|
97
99
|
const alreadyHooked = type === "event" || type === "user" || type === "scd";
|
|
98
100
|
|
|
99
101
|
// Performance optimization: skip hook overhead for passthrough hooks
|
|
100
102
|
// Only treat as passthrough if the function body is trivially simple (just returns its argument)
|
|
101
103
|
const hookStr = hook.toString();
|
|
102
|
-
const isPassthroughHook = hook.length === 1 || /^\s*function\s*\([^)]*\)\s*\{\s*return\s+\w+;?\s*\}\s*$/.test(hookStr) || /^\s*\(?[^)]*\)?\s*=>\s*\w+\s*$/.test(hookStr);
|
|
104
|
+
const isPassthroughHook = !isWarehouse && (hook.length === 1 || /^\s*function\s*\([^)]*\)\s*\{\s*return\s+\w+;?\s*\}\s*$/.test(hookStr) || /^\s*\(?[^)]*\)?\s*=>\s*\w+\s*$/.test(hookStr));
|
|
103
105
|
|
|
104
106
|
if (alreadyHooked || isPassthroughHook) {
|
|
105
107
|
// Fast path for passthrough hooks - no transformation needed
|
|
@@ -131,6 +133,10 @@ export async function createHookArray(arr = [], opts) {
|
|
|
131
133
|
for (const i of item) {
|
|
132
134
|
try {
|
|
133
135
|
const enriched = await hook(i, type, allMetaData);
|
|
136
|
+
if (isWarehouse) {
|
|
137
|
+
if (isValidEvent(i)) arr.push(i);
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
134
140
|
if (Array.isArray(enriched)) {
|
|
135
141
|
enriched.forEach(e => {
|
|
136
142
|
if (isValidEvent(e)) arr.push(e);
|
|
@@ -146,6 +152,10 @@ export async function createHookArray(arr = [], opts) {
|
|
|
146
152
|
} else {
|
|
147
153
|
try {
|
|
148
154
|
const enriched = await hook(item, type, allMetaData);
|
|
155
|
+
if (isWarehouse) {
|
|
156
|
+
if (isValidEvent(item)) arr.push(item);
|
|
157
|
+
return Promise.resolve(false);
|
|
158
|
+
}
|
|
149
159
|
if (Array.isArray(enriched)) {
|
|
150
160
|
enriched.forEach(e => {
|
|
151
161
|
if (isValidEvent(e)) arr.push(e);
|
|
@@ -161,7 +171,7 @@ export async function createHookArray(arr = [], opts) {
|
|
|
161
171
|
}
|
|
162
172
|
|
|
163
173
|
// Check batch size and handle writes synchronously to prevent race conditions
|
|
164
|
-
if (arr.length > BATCH_SIZE && !isWriting) {
|
|
174
|
+
if (!isWarehouse && arr.length > BATCH_SIZE && !isWriting) {
|
|
165
175
|
isWriting = true; // Lock to prevent concurrent writes
|
|
166
176
|
isBatchMode = true;
|
|
167
177
|
runtime.isBatchMode = true; // Update runtime state
|
|
@@ -200,6 +210,9 @@ export async function createHookArray(arr = [], opts) {
|
|
|
200
210
|
const streamOptions = {
|
|
201
211
|
gzip: config.gzip || false
|
|
202
212
|
};
|
|
213
|
+
if (type === "warehouse" && Array.isArray(rest.fixedColumns)) {
|
|
214
|
+
streamOptions.fixedColumns = rest.fixedColumns;
|
|
215
|
+
}
|
|
203
216
|
|
|
204
217
|
switch (format) {
|
|
205
218
|
case "csv":
|
|
@@ -253,6 +266,8 @@ export async function createHookArray(arr = [], opts) {
|
|
|
253
266
|
enrichedArray.getWriteDir = getWriteDir;
|
|
254
267
|
enrichedArray.getWritePath = getWritePath;
|
|
255
268
|
enrichedArray.getWrittenFiles = () => [...writtenFiles];
|
|
269
|
+
enrichedArray.type = type;
|
|
270
|
+
enrichedArray.format = format;
|
|
256
271
|
|
|
257
272
|
// Add additional properties from rest
|
|
258
273
|
for (const key in rest) {
|
|
@@ -309,9 +324,20 @@ export class StorageManager {
|
|
|
309
324
|
context: this.context
|
|
310
325
|
}),
|
|
311
326
|
|
|
327
|
+
// v1.8.0 — identity-less metric snapshots (`standaloneEvents`).
|
|
328
|
+
standaloneEventData: await createHookArray([], {
|
|
329
|
+
hook: config.hook,
|
|
330
|
+
type: "standalone",
|
|
331
|
+
filepath: `${config.name}-STANDALONE`,
|
|
332
|
+
format: config.format || "csv",
|
|
333
|
+
concurrency: config.concurrency || 1,
|
|
334
|
+
context: this.context
|
|
335
|
+
}),
|
|
336
|
+
|
|
312
337
|
scdTableData: [],
|
|
313
338
|
groupProfilesData: [],
|
|
314
339
|
lookupTableData: [],
|
|
340
|
+
warehouseMetricData: [],
|
|
315
341
|
|
|
316
342
|
mirrorEventData: await createHookArray([], {
|
|
317
343
|
hook: config.hook,
|
|
@@ -375,6 +401,28 @@ export class StorageManager {
|
|
|
375
401
|
}
|
|
376
402
|
}
|
|
377
403
|
|
|
404
|
+
if (config.warehouseMetrics && config.warehouseMetrics.length > 0) {
|
|
405
|
+
for (const warehouseMetric of config.warehouseMetrics) {
|
|
406
|
+
const fixedColumns = [
|
|
407
|
+
warehouseMetric.timeColumn,
|
|
408
|
+
...(warehouseMetric.source.groupBy || []),
|
|
409
|
+
warehouseMetric.valueColumn,
|
|
410
|
+
...Object.keys(warehouseMetric.columns || {}),
|
|
411
|
+
];
|
|
412
|
+
const warehouseArray = await createHookArray([], {
|
|
413
|
+
hook: config.hook,
|
|
414
|
+
type: "warehouse",
|
|
415
|
+
filepath: `${config.name}-WAREHOUSE-${warehouseMetric.name}`,
|
|
416
|
+
format: warehouseMetric.format || config.format || "csv",
|
|
417
|
+
concurrency: config.concurrency || 1,
|
|
418
|
+
context: this.context,
|
|
419
|
+
metricName: warehouseMetric.name,
|
|
420
|
+
fixedColumns,
|
|
421
|
+
});
|
|
422
|
+
storage.warehouseMetricData.push(warehouseArray);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
378
426
|
return storage;
|
|
379
427
|
}
|
|
380
428
|
|