@ak--47/dungeon-master 1.6.5 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -11
- package/.claude/skills/create-dungeon/SKILL.md +49 -10
- package/.claude/skills/create-project/SKILL.md +22 -3
- package/.claude/skills/create-project/context.mjs +89 -0
- package/.claude/skills/create-project/provision.mjs +1 -60
- package/.claude/skills/headless-build/SKILL.md +18 -1
- package/.claude/skills/powertools/SKILL.md +20 -1
- package/.claude/skills/release-check/SKILL.md +99 -0
- package/.claude/skills/verify-dungeon/SKILL.md +71 -16
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +14 -0
- package/.claude/skills/verify-dungeon/references/report-format.md +18 -1
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +36 -1
- package/.claude/skills/warehouse-metrics/GAPS-template.md +34 -0
- package/.claude/skills/warehouse-metrics/SKILL.md +105 -0
- package/.claude/skills/warehouse-metrics/deploy.mjs +651 -0
- package/.claude/skills/write-hooks/SKILL.md +33 -3
- package/CHANGELOG.md +331 -0
- package/HOOKS.md +154 -5
- package/README.md +357 -8
- package/docs/guides/1.7.0-upgrade-guide.md +154 -0
- package/docs/guides/1.8.0-upgrade-guide.md +151 -0
- package/dungeons/technical/warehouse.js +187 -0
- package/index.js +131 -2
- package/lib/core/config-validator.js +264 -13
- package/lib/core/context.js +39 -0
- package/lib/core/dungeon-loader.js +5 -2
- package/lib/core/storage.js +51 -3
- package/lib/generators/events.js +53 -7
- package/lib/generators/funnels.js +85 -11
- package/lib/generators/profiles.js +9 -4
- package/lib/generators/standalone.js +248 -0
- package/lib/generators/warehouse.js +828 -0
- package/lib/orchestrators/mixpanel-sender.js +39 -3
- package/lib/orchestrators/user-loop.js +240 -9
- package/lib/templates/story-spec.schema.json +41 -16
- package/lib/utils/conditions.js +62 -0
- package/lib/utils/json-evaluator.js +12 -2
- package/lib/utils/utils.js +115 -19
- package/lib/verify/index.js +1 -0
- package/lib/verify/schema-validator.js +8 -0
- package/lib/verify/story-runner.js +71 -8
- package/lib/verify/warehouse.js +683 -0
- package/package.json +5 -11
- package/scripts/verify-stories.mjs +150 -44
- package/types.d.ts +606 -38
|
@@ -71,6 +71,7 @@ async function _sendToMixpanel(context) {
|
|
|
71
71
|
const { config, storage } = context;
|
|
72
72
|
const {
|
|
73
73
|
adSpendData,
|
|
74
|
+
standaloneEventData,
|
|
74
75
|
eventData,
|
|
75
76
|
groupProfilesData,
|
|
76
77
|
scdTableData,
|
|
@@ -175,7 +176,18 @@ async function _sendToMixpanel(context) {
|
|
|
175
176
|
progressCallback: makeProgressCallback(userTotal),
|
|
176
177
|
});
|
|
177
178
|
log(` -> ${comma(imported.success)} user profiles sent\n`);
|
|
178
|
-
|
|
179
|
+
// v1.7.0 (R2-5): make the receipt self-explanatory. `generated` counts every
|
|
180
|
+
// profile the engine pushed to storage (bots included); `dropped_anonymous`
|
|
181
|
+
// counts the `_drop`-flagged anonymous non-converters that never reach
|
|
182
|
+
// /engage. `generated - dropped_anonymous - failed === success` is checkable.
|
|
183
|
+
// Counters live on the context and tick at push time, so they hold in
|
|
184
|
+
// batch mode where the in-memory array has been flushed.
|
|
185
|
+
const generated = context.runtime.profilesGenerated;
|
|
186
|
+
const dropped_anonymous = context.runtime.profilesDropped;
|
|
187
|
+
importResults.users = { ...imported, generated, dropped_anonymous };
|
|
188
|
+
if (imported.success + (imported.failed || 0) + dropped_anonymous !== generated) {
|
|
189
|
+
log(` !! profile receipt does not reconcile: ${comma(generated)} generated - ${comma(dropped_anonymous)} dropped anonymous - ${comma(imported.failed || 0)} failed != ${comma(imported.success)} success\n`);
|
|
190
|
+
}
|
|
179
191
|
}
|
|
180
192
|
|
|
181
193
|
// Import ad spend data
|
|
@@ -197,6 +209,29 @@ async function _sendToMixpanel(context) {
|
|
|
197
209
|
importResults.adSpend = imported;
|
|
198
210
|
}
|
|
199
211
|
|
|
212
|
+
// Import standalone (identity-less) metric snapshots — v1.8.0.
|
|
213
|
+
// Same shape as the ad-spend path: its own stream, imported as events.
|
|
214
|
+
// Gated on the config so a batch-mode run without `standaloneEvents` does
|
|
215
|
+
// not attempt an empty file read.
|
|
216
|
+
const hasStandaloneConfig = Array.isArray(config.standaloneEvents) && config.standaloneEvents.length > 0;
|
|
217
|
+
if (hasStandaloneConfig && (standaloneEventData?.length > 0 || isBATCH_MODE)) {
|
|
218
|
+
log(` Standalone Events`);
|
|
219
|
+
let standaloneToImport = u.deepClone(standaloneEventData);
|
|
220
|
+
const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && standaloneEventData && standaloneEventData.length === 0);
|
|
221
|
+
if (shouldReadFromFiles && standaloneEventData?.getWrittenFiles) {
|
|
222
|
+
const files = standaloneEventData.getWrittenFiles();
|
|
223
|
+
if (files.length > 0) standaloneToImport = files;
|
|
224
|
+
}
|
|
225
|
+
const standaloneTotal = Array.isArray(standaloneToImport) ? standaloneToImport.length : 0;
|
|
226
|
+
const imported = await mp(creds, standaloneToImport, {
|
|
227
|
+
recordType: "event",
|
|
228
|
+
...commonOpts,
|
|
229
|
+
progressCallback: makeProgressCallback(standaloneTotal),
|
|
230
|
+
});
|
|
231
|
+
log(` -> ${comma(imported.success)} standalone events sent\n`);
|
|
232
|
+
importResults.standalone = imported;
|
|
233
|
+
}
|
|
234
|
+
|
|
200
235
|
// Import group profiles
|
|
201
236
|
if (groupProfilesData && Array.isArray(groupProfilesData) && groupProfilesData.length > 0) {
|
|
202
237
|
for (const groupEntity of groupProfilesData) {
|
|
@@ -392,11 +427,12 @@ function logProblems(problems) {
|
|
|
392
427
|
*/
|
|
393
428
|
function collectWrittenFiles(storage) {
|
|
394
429
|
const files = [];
|
|
430
|
+
if (storage.warehouseManifestFile) files.push(storage.warehouseManifestFile);
|
|
395
431
|
for (const container of [storage.eventData, storage.userProfilesData, storage.adSpendData,
|
|
396
|
-
storage.mirrorEventData, storage.groupEventData]) {
|
|
432
|
+
storage.standaloneEventData, storage.mirrorEventData, storage.groupEventData]) {
|
|
397
433
|
if (container?.getWrittenFiles) files.push(...container.getWrittenFiles());
|
|
398
434
|
}
|
|
399
|
-
for (const arr of [storage.groupProfilesData, storage.scdTableData, storage.lookupTableData]) {
|
|
435
|
+
for (const arr of [storage.groupProfilesData, storage.scdTableData, storage.lookupTableData, storage.warehouseMetricData]) {
|
|
400
436
|
if (Array.isArray(arr)) {
|
|
401
437
|
for (const c of arr) {
|
|
402
438
|
if (c?.getWrittenFiles) files.push(...c.getWrittenFiles());
|
|
@@ -16,6 +16,11 @@ import { makeFunnel } from "../generators/funnels.js";
|
|
|
16
16
|
import { makeUserProfile } from "../generators/profiles.js";
|
|
17
17
|
import { makeSCD } from "../generators/scd.js";
|
|
18
18
|
import { buildCurveWeightFn, expectedActiveDays } from "../utils/retention-curve.js";
|
|
19
|
+
import { matchConditions } from "../utils/conditions.js";
|
|
20
|
+
|
|
21
|
+
// v1.7.0 (P0-1): matching moved to lib/utils/conditions.js (operator maps).
|
|
22
|
+
// Re-exported here so existing imports keep working.
|
|
23
|
+
export { matchConditions };
|
|
19
24
|
|
|
20
25
|
/**
|
|
21
26
|
* Main user generation loop that creates users, their profiles, events, and SCDs
|
|
@@ -69,6 +74,23 @@ export async function userLoop(context) {
|
|
|
69
74
|
// Track if we've already logged the strict event count message
|
|
70
75
|
let hasLoggedStrictCountReached = false;
|
|
71
76
|
|
|
77
|
+
// v1.7.0 (R2-3): strictEventCount bookkeeping. `strictBudgetSum` is the sum
|
|
78
|
+
// of per-user budgets handed out so far; with the STORED count it yields the
|
|
79
|
+
// realized events-per-budget ratio, which the controller below uses to scale
|
|
80
|
+
// the remaining users' budgets so the run lands on `numEvents` and the final
|
|
81
|
+
// trim makes it exact. Before 1.7.0 the flag stopped on the GENERATED count
|
|
82
|
+
// (which includes `_drop`'d events) and never topped up, so it delivered ~6%
|
|
83
|
+
// short with 4x headroom available.
|
|
84
|
+
let strictBudgetSum = 0;
|
|
85
|
+
let strictUsersBudgeted = 0;
|
|
86
|
+
|
|
87
|
+
// v1.7.0 run-level counters for aggregated warnings (P0-1 item 5, P1-5).
|
|
88
|
+
const anyConditionedFunnel = Array.isArray(funnels) && funnels.some(f => f && f.conditions);
|
|
89
|
+
let usersMatchingNoFunnel = 0;
|
|
90
|
+
let usersChurned = 0;
|
|
91
|
+
const personaMultipliersInPlay = Array.isArray(personas) && personas.some(p => p && Number.isFinite(p.eventMultiplier) && p.eventMultiplier !== 1);
|
|
92
|
+
const campaignKeys = ['utm_source', 'utm_campaign', 'utm_medium', 'utm_content', 'utm_term'];
|
|
93
|
+
|
|
72
94
|
// Handle graceful shutdown on SIGINT (Ctrl+C).
|
|
73
95
|
// CRITICAL: listener MUST be removed in a `finally` block — pre-fix, throws /
|
|
74
96
|
// cancellation paths leaked listeners across test runs, accumulating until
|
|
@@ -87,8 +109,9 @@ export async function userLoop(context) {
|
|
|
87
109
|
// Bail out if cancelled
|
|
88
110
|
if (cancelled) return;
|
|
89
111
|
|
|
90
|
-
// Bail out early if strictEventCount is enabled and we've hit numEvents
|
|
91
|
-
|
|
112
|
+
// Bail out early if strictEventCount is enabled and we've hit numEvents.
|
|
113
|
+
// v1.7.0: compares the STORED count (post-drop), not the generated count.
|
|
114
|
+
if (strictEventCount && context.getStoredEventCount() >= numEvents) {
|
|
92
115
|
if (verbose && !hasLoggedStrictCountReached) {
|
|
93
116
|
console.log(`\n\u2713 Reached target of ${numEvents.toLocaleString()} events with strict event count enabled. Stopping user generation.`);
|
|
94
117
|
hasLoggedStrictCountReached = true;
|
|
@@ -213,17 +236,40 @@ export async function userLoop(context) {
|
|
|
213
236
|
// Feature 1: Merge persona properties into profile (before hook, so hook can override)
|
|
214
237
|
if (persona && persona.properties) {
|
|
215
238
|
for (const [key, value] of Object.entries(persona.properties)) {
|
|
216
|
-
profile[key] = u.choose(value);
|
|
239
|
+
profile[key] = u.choose(value, { profile, config });
|
|
217
240
|
}
|
|
218
241
|
profile._persona = persona.name;
|
|
219
242
|
}
|
|
220
243
|
|
|
244
|
+
// v1.7.0 (P1-4): one acquisition campaign per user. Draw a campaign
|
|
245
|
+
// template at birth and stamp its UTM keys on the profile. Any UTM key the
|
|
246
|
+
// profile already carries (persona `properties`) wins over the draw, so
|
|
247
|
+
// "the paid-search persona" is `properties: { utm_source: 'google' }`. The
|
|
248
|
+
// `user` hook runs after this and can override too; the touchpoint pass
|
|
249
|
+
// reads the FINAL profile values. Opt-in: no RNG draw unless enabled.
|
|
250
|
+
if (config.campaignPerUser && config.hasCampaigns) {
|
|
251
|
+
const template = u.pickRandom(defaults.campaigns());
|
|
252
|
+
if (template && typeof template === 'object') {
|
|
253
|
+
for (const key of campaignKeys) {
|
|
254
|
+
if (profile[key] !== undefined) continue;
|
|
255
|
+
if (template[key] === undefined) continue;
|
|
256
|
+
profile[key] = u.choose(template[key], { profile, config });
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
221
261
|
// Build feature context for event generation
|
|
222
262
|
const featureCtx = {
|
|
223
263
|
persona,
|
|
224
264
|
userLocation,
|
|
225
265
|
worldEventsTimeline: worldEvents,
|
|
226
266
|
dataQuality,
|
|
267
|
+
// v1.7.0: value context (P1-1), sticky event props (P1-2), per-user
|
|
268
|
+
// campaign (P1-4), experiment assignments collector (P0-2).
|
|
269
|
+
profile,
|
|
270
|
+
stickyValues: null,
|
|
271
|
+
userCampaign: null,
|
|
272
|
+
experimentAssignments: new Map(),
|
|
227
273
|
};
|
|
228
274
|
|
|
229
275
|
// Call user hook after profile creation (hooks override persona properties).
|
|
@@ -246,6 +292,32 @@ export async function userLoop(context) {
|
|
|
246
292
|
}
|
|
247
293
|
}
|
|
248
294
|
|
|
295
|
+
// v1.7.0 (P1-2): resolve sticky event props ONCE per user, after the user
|
|
296
|
+
// hook so hook overrides are honored. Keys declared in userProps / persona
|
|
297
|
+
// properties copy from the profile; keys declared only in superProps are
|
|
298
|
+
// resolved here once (instead of per event) and held constant.
|
|
299
|
+
if (Array.isArray(config.stickyEventProps) && config.stickyEventProps.length) {
|
|
300
|
+
const superOnly = new Set(config._stickySuperOnly || []);
|
|
301
|
+
const stickyValues = {};
|
|
302
|
+
for (const key of config.stickyEventProps) {
|
|
303
|
+
stickyValues[key] = superOnly.has(key)
|
|
304
|
+
? u.choose(config.superProps[key], { profile, config })
|
|
305
|
+
: profile[key];
|
|
306
|
+
}
|
|
307
|
+
featureCtx.stickyValues = stickyValues;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// v1.7.0 (P1-4): the user's campaign, read from the FINAL profile so persona
|
|
311
|
+
// properties and the user hook both flow into the touchpoint pass.
|
|
312
|
+
if (config.campaignPerUser && config.hasCampaigns) {
|
|
313
|
+
const userCampaign = {};
|
|
314
|
+
let any = false;
|
|
315
|
+
for (const key of campaignKeys) {
|
|
316
|
+
if (profile[key] !== undefined && profile[key] !== null) { userCampaign[key] = profile[key]; any = true; }
|
|
317
|
+
}
|
|
318
|
+
featureCtx.userCampaign = any ? userCampaign : null;
|
|
319
|
+
}
|
|
320
|
+
|
|
249
321
|
// SCD creation
|
|
250
322
|
// @ts-ignore
|
|
251
323
|
const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
|
|
@@ -305,6 +377,31 @@ export async function userLoop(context) {
|
|
|
305
377
|
}
|
|
306
378
|
numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
|
|
307
379
|
|
|
380
|
+
// v1.7.0 (R2-3): strictEventCount budget controller. Born-in-dataset users
|
|
381
|
+
// carry a pro-rated budget and `_drop`s thin the rest, so the raw budgets
|
|
382
|
+
// sum below `numEvents` by design (~6% measured). Scale this user's budget
|
|
383
|
+
// by (events still needed) / (events the remaining users would deliver at
|
|
384
|
+
// the realized yield), aiming 2% high so the final per-user trim has
|
|
385
|
+
// something to trim. Clamped to [0.25, 4] so no single user is distorted
|
|
386
|
+
// into a whale. Only fires under the flag — the default path is untouched.
|
|
387
|
+
if (strictEventCount && numEventsThisUserWillPreform > 0) {
|
|
388
|
+
const delivered = context.getStoredEventCount();
|
|
389
|
+
const remainingTarget = numEvents - delivered;
|
|
390
|
+
const remainingUsers = Math.max(1, numUsers - i);
|
|
391
|
+
const meanBudget = strictUsersBudgeted > 0 ? strictBudgetSum / strictUsersBudgeted : numEventsThisUserWillPreform;
|
|
392
|
+
const yieldRatio = strictBudgetSum > 0 ? delivered / strictBudgetSum : 1;
|
|
393
|
+
const expectedRemaining = meanBudget * yieldRatio * remainingUsers;
|
|
394
|
+
if (remainingTarget > 0 && expectedRemaining > 0) {
|
|
395
|
+
// Overshoot grows as the tail shrinks (last user aims 2x its share) so a
|
|
396
|
+
// low-yield final user cannot leave the run short; the trim removes excess.
|
|
397
|
+
const overshoot = 1.02 + 1 / remainingUsers;
|
|
398
|
+
const scale = Math.max(0.25, Math.min(4, (remainingTarget * overshoot) / expectedRemaining));
|
|
399
|
+
numEventsThisUserWillPreform = Math.max(1, Math.round(numEventsThisUserWillPreform * scale));
|
|
400
|
+
}
|
|
401
|
+
strictBudgetSum += numEventsThisUserWillPreform;
|
|
402
|
+
strictUsersBudgeted++;
|
|
403
|
+
}
|
|
404
|
+
|
|
308
405
|
let userFirstEventTime;
|
|
309
406
|
|
|
310
407
|
// ── v1.5 Active-day scheduling ──
|
|
@@ -354,6 +451,12 @@ export async function userLoop(context) {
|
|
|
354
451
|
const usageFunnels = funnels.filter((f) => !f.isFirstFunnel)
|
|
355
452
|
.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
|
|
356
453
|
.reduce(weighFunnels, []);
|
|
454
|
+
// v1.7.0 (P0-1 item 5): with conditions in play, a user who matches no
|
|
455
|
+
// funnel at all silently falls through to standalone events. Counted here,
|
|
456
|
+
// reported once per run (see end of loop).
|
|
457
|
+
if (anyConditionedFunnel && firstFunnels.length === 0 && !usageFunnels.some(f => !f._catchAll)) {
|
|
458
|
+
usersMatchingNoFunnel++;
|
|
459
|
+
}
|
|
357
460
|
|
|
358
461
|
const secondsInDay = 86400;
|
|
359
462
|
const noise = () => chance.integer({ min: 0, max: secondsInDay });
|
|
@@ -627,6 +730,24 @@ export async function userLoop(context) {
|
|
|
627
730
|
return Number.isFinite(t) ? t <= userChurnTimeMs : true;
|
|
628
731
|
});
|
|
629
732
|
}
|
|
733
|
+
if (userChurned) usersChurned++;
|
|
734
|
+
|
|
735
|
+
// v1.7.0 (P0-3): world-event amplification. `volumeMultiplier > 1` clones
|
|
736
|
+
// affected in-window events (fresh insert_id, spread uniformly across the
|
|
737
|
+
// window) so daily volume reaches the stated multiple. Runs after the churn
|
|
738
|
+
// cut (no post-churn clones) and before decay / hooks, which see the clones.
|
|
739
|
+
const resolvedWorldEvents = /** @type {import('../../types').ResolvedWorldEvent[] | null} */ (worldEvents);
|
|
740
|
+
if (resolvedWorldEvents && usersEvents.length > 0 && worldEventsAmplify(resolvedWorldEvents)) {
|
|
741
|
+
usersEvents = amplifyWorldEvents(usersEvents, resolvedWorldEvents, chance, context.FIXED_NOW);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// v1.7.0 (P0-2): stamp sticky experiment assignments on the profile as
|
|
745
|
+
// `Experiment: <name>` before the everything hook, so hooks see it too.
|
|
746
|
+
if (featureCtx.experimentAssignments.size > 0) {
|
|
747
|
+
for (const [expName, variantName] of featureCtx.experimentAssignments) {
|
|
748
|
+
profile[`Experiment: ${expName}`] = variantName;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
630
751
|
|
|
631
752
|
// Feature 3: Engagement decay — filter behavioral events
|
|
632
753
|
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
@@ -696,7 +817,7 @@ export async function userLoop(context) {
|
|
|
696
817
|
// preserves realistic touch shape — last-10-window attribution then
|
|
697
818
|
// gives meaningful first/last-touch results.
|
|
698
819
|
if (config.hasCampaigns && usersEvents.length > 0) {
|
|
699
|
-
applyTouchpointCap(usersEvents, config, defaults, chance);
|
|
820
|
+
applyTouchpointCap(usersEvents, config, defaults, chance, featureCtx.userCampaign);
|
|
700
821
|
}
|
|
701
822
|
|
|
702
823
|
// v1.5.1: anonymous non-converters never call $identify in production,
|
|
@@ -819,9 +940,25 @@ export async function userLoop(context) {
|
|
|
819
940
|
u.assignSessionIds(sortedView, sessionTimeout);
|
|
820
941
|
}
|
|
821
942
|
|
|
943
|
+
// v1.7.0 (R2-3): strict mode never exceeds numEvents. Trim this user's
|
|
944
|
+
// stream to the remaining room with a seeded uniform sample (keeps the
|
|
945
|
+
// time shape; preserves array order). Only under the flag.
|
|
946
|
+
if (strictEventCount && usersEvents.length > 0) {
|
|
947
|
+
const room = numEvents - context.getStoredEventCount();
|
|
948
|
+
if (room <= 0) {
|
|
949
|
+
usersEvents = [];
|
|
950
|
+
} else if (usersEvents.length > room) {
|
|
951
|
+
const keep = new Set(chance.pickset(usersEvents, room));
|
|
952
|
+
usersEvents = usersEvents.filter(e => keep.has(e));
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
|
|
822
956
|
// Store all user data (skip profile push when a hook returned null
|
|
823
957
|
// for type='user' — see dropUserProfile above).
|
|
824
958
|
if (!dropUserProfile) {
|
|
959
|
+
// v1.7.0 (R2-5): receipt counters tick at push time (batch-mode safe).
|
|
960
|
+
context.incrementProfilesGenerated();
|
|
961
|
+
if (profile._drop) context.incrementProfilesDropped();
|
|
825
962
|
await userProfilesData.hookPush(profile);
|
|
826
963
|
}
|
|
827
964
|
|
|
@@ -841,6 +978,7 @@ export async function userLoop(context) {
|
|
|
841
978
|
}
|
|
842
979
|
}
|
|
843
980
|
|
|
981
|
+
context.warehouseAccumulator?.ingest(usersEvents);
|
|
844
982
|
await eventData.hookPush(usersEvents, { profile });
|
|
845
983
|
});
|
|
846
984
|
|
|
@@ -868,6 +1006,42 @@ export async function userLoop(context) {
|
|
|
868
1006
|
if (dataQuality && dataQuality.botUsers > 0) {
|
|
869
1007
|
await generateBotUsers(context, dataQuality, storage);
|
|
870
1008
|
}
|
|
1009
|
+
|
|
1010
|
+
// ── v1.7.0 aggregated run-level warnings (surfaced as result.warnings) ──
|
|
1011
|
+
if (typeof context.addWarning === 'function') {
|
|
1012
|
+
if (usersMatchingNoFunnel > 0) {
|
|
1013
|
+
context.addWarning({
|
|
1014
|
+
key: 'funnels.conditions',
|
|
1015
|
+
requested: numUsers,
|
|
1016
|
+
applied: numUsers - usersMatchingNoFunnel,
|
|
1017
|
+
reason: `${usersMatchingNoFunnel} of ${numUsers} users matched no funnel's conditions and generated standalone events only; check that the conditioned funnels cover every segment`,
|
|
1018
|
+
severity: 'warn',
|
|
1019
|
+
count: usersMatchingNoFunnel,
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
if (personaMultipliersInPlay && numUsers > 0 && usersChurned / numUsers > 0.5) {
|
|
1023
|
+
context.addWarning({
|
|
1024
|
+
key: 'personas.eventMultiplier',
|
|
1025
|
+
requested: usersChurned,
|
|
1026
|
+
applied: numUsers,
|
|
1027
|
+
reason: `${usersChurned} of ${numUsers} users were ended by an isChurnEvent; churn caps every user's lifetime event count, so personas[].eventMultiplier cannot lift a persona past it (measured 1.04x for an asked 3x). Lower the churn event's weight, raise returnLikelihood, or drive churn from a hook.`,
|
|
1028
|
+
severity: 'warn',
|
|
1029
|
+
count: usersChurned,
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
if (strictEventCount) {
|
|
1033
|
+
const stored = context.getStoredEventCount();
|
|
1034
|
+
if (stored < numEvents) {
|
|
1035
|
+
context.addWarning({
|
|
1036
|
+
key: 'numEvents',
|
|
1037
|
+
requested: numEvents,
|
|
1038
|
+
applied: stored,
|
|
1039
|
+
reason: `strictEventCount could not reach numEvents: the ${numUsers} users' capacity (rate × active days, minus drops) ran out ${numEvents - stored} events short. Raise numUsers, numDays, or avgEventsPerUserPerDay.`,
|
|
1040
|
+
severity: 'warn',
|
|
1041
|
+
});
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
871
1045
|
} finally {
|
|
872
1046
|
// Always remove the SIGINT listener — even if userLoop throws or is
|
|
873
1047
|
// cancelled. Pre-fix this leaked across test runs and stalled workers.
|
|
@@ -884,11 +1058,61 @@ export function weighFunnels(acc, funnel) {
|
|
|
884
1058
|
return acc;
|
|
885
1059
|
}
|
|
886
1060
|
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
1061
|
+
/**
|
|
1062
|
+
* v1.7.0 (P0-3): true when any resolved world event asks for amplification.
|
|
1063
|
+
* @param {import('../../types').ResolvedWorldEvent[]} worldEvents
|
|
1064
|
+
*/
|
|
1065
|
+
function worldEventsAmplify(worldEvents) {
|
|
1066
|
+
return worldEvents.some(we => we && ((we.volumeMultiplier > 1) || (we.aftermath && we.aftermath.volumeMultiplier > 1)));
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
/**
|
|
1070
|
+
* v1.7.0 (P0-3): clone affected events so volume inside a world-event window (or
|
|
1071
|
+
* its aftermath) reaches `volumeMultiplier`× the baseline.
|
|
1072
|
+
*
|
|
1073
|
+
* For every event that falls inside an amplifying window and matches
|
|
1074
|
+
* `affectsEvents`, add `floor(mult - 1)` clones plus one more with probability
|
|
1075
|
+
* `frac(mult)` — so `2.5` means one guaranteed clone and a 50% second. Each clone
|
|
1076
|
+
* gets a FRESH `insert_id` (Mixpanel dedupes on it — a spread that keeps the
|
|
1077
|
+
* source id is silently dropped at ingest) and a timestamp drawn uniformly across
|
|
1078
|
+
* the window, never past `fixedNow`, so the surge is spread over the window's
|
|
1079
|
+
* days rather than stacked one second after its source. Mutates nothing; returns
|
|
1080
|
+
* a new array. Deterministic under the seeded chance.
|
|
1081
|
+
*
|
|
1082
|
+
* @param {Object[]} events - the user's surviving events
|
|
1083
|
+
* @param {import('../../types').ResolvedWorldEvent[]} worldEvents
|
|
1084
|
+
* @param {Object} chance - seeded chance instance
|
|
1085
|
+
* @param {number} fixedNow - dataset end (unix seconds)
|
|
1086
|
+
* @returns {Object[]}
|
|
1087
|
+
*/
|
|
1088
|
+
export function amplifyWorldEvents(events, worldEvents, chance, fixedNow) {
|
|
1089
|
+
const clones = [];
|
|
1090
|
+
for (const ev of events) {
|
|
1091
|
+
if (!ev || !ev.time) continue;
|
|
1092
|
+
const evUnix = Math.floor(Date.parse(ev.time) / 1000);
|
|
1093
|
+
if (!Number.isFinite(evUnix)) continue;
|
|
1094
|
+
for (const we of worldEvents) {
|
|
1095
|
+
if (!we) continue;
|
|
1096
|
+
const inMain = evUnix >= we.startUnix && evUnix < we.endUnix;
|
|
1097
|
+
const inAftermath = !!we.aftermathEndUnix && evUnix >= we.endUnix && evUnix < we.aftermathEndUnix;
|
|
1098
|
+
if (!inMain && !inAftermath) continue;
|
|
1099
|
+
const affects = we.affectsEvents;
|
|
1100
|
+
if (!(affects === "*" || (Array.isArray(affects) && affects.includes(ev.event)))) continue;
|
|
1101
|
+
const mult = inMain ? we.volumeMultiplier : (we.aftermath && we.aftermath.volumeMultiplier) || 1;
|
|
1102
|
+
if (!(mult > 1)) continue;
|
|
1103
|
+
const windowStart = inMain ? we.startUnix : we.endUnix;
|
|
1104
|
+
const windowEnd = Math.min(inMain ? we.endUnix : we.aftermathEndUnix, fixedNow);
|
|
1105
|
+
if (!(windowEnd > windowStart)) continue;
|
|
1106
|
+
let copies = Math.floor(mult - 1);
|
|
1107
|
+
const frac = mult - 1 - copies;
|
|
1108
|
+
if (frac > 0 && chance.bool({ likelihood: frac * 100 })) copies++;
|
|
1109
|
+
for (let c = 0; c < copies; c++) {
|
|
1110
|
+
const t = chance.integer({ min: windowStart, max: windowEnd - 1 });
|
|
1111
|
+
clones.push({ ...ev, time: dayjs.unix(t).toISOString(), insert_id: randomUUID() });
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
890
1114
|
}
|
|
891
|
-
return
|
|
1115
|
+
return clones.length ? events.concat(clones) : events;
|
|
892
1116
|
}
|
|
893
1117
|
|
|
894
1118
|
// ── v1.5 Active-Day Plan Helpers ──
|
|
@@ -1058,8 +1282,11 @@ function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDa
|
|
|
1058
1282
|
* @param {Object} config - Validated dungeon config
|
|
1059
1283
|
* @param {Object} defaults - Context.defaults (provides campaigns())
|
|
1060
1284
|
* @param {Object} chance - Seeded chance instance
|
|
1285
|
+
* @param {Object|null} [userCampaign] - v1.7.0 (P1-4): the user's resolved UTM values
|
|
1286
|
+
* (from the profile) when `campaignPerUser` is on. Stamped verbatim on every
|
|
1287
|
+
* sampled touchpoint instead of re-drawing a template per event.
|
|
1061
1288
|
*/
|
|
1062
|
-
function applyTouchpointCap(events, config, defaults, chance) {
|
|
1289
|
+
function applyTouchpointCap(events, config, defaults, chance, userCampaign = null) {
|
|
1063
1290
|
const cap = Number.isFinite(config.maxTouchpointsPerUser)
|
|
1064
1291
|
? config.maxTouchpointsPerUser
|
|
1065
1292
|
: 10;
|
|
@@ -1101,6 +1328,10 @@ function applyTouchpointCap(events, config, defaults, chance) {
|
|
|
1101
1328
|
|
|
1102
1329
|
// Stamp UTMs on each sampled event using a campaign template.
|
|
1103
1330
|
for (const ev of sample) {
|
|
1331
|
+
if (userCampaign) {
|
|
1332
|
+
for (const [k, v] of Object.entries(userCampaign)) ev[k] = v;
|
|
1333
|
+
continue;
|
|
1334
|
+
}
|
|
1104
1335
|
const campaignTemplate = u.pickRandom(defaults.campaigns());
|
|
1105
1336
|
if (!campaignTemplate || typeof campaignTemplate !== 'object') continue;
|
|
1106
1337
|
for (const [k, v] of Object.entries(campaignTemplate)) {
|
|
@@ -85,7 +85,6 @@
|
|
|
85
85
|
"properties": {
|
|
86
86
|
"where": {
|
|
87
87
|
"type": "object",
|
|
88
|
-
"minProperties": 1,
|
|
89
88
|
"description": "Column → value (equality) or { op, value } comparison. All clauses must match (AND).",
|
|
90
89
|
"additionalProperties": {
|
|
91
90
|
"oneOf": [
|
|
@@ -123,23 +122,49 @@
|
|
|
123
122
|
"properties": {
|
|
124
123
|
"type": { "type": "string", "minLength": 1 }
|
|
125
124
|
},
|
|
126
|
-
"
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
125
|
+
"allOf": [
|
|
126
|
+
{
|
|
127
|
+
"if": {
|
|
128
|
+
"properties": { "type": { "const": "duckdb" } }
|
|
129
|
+
},
|
|
130
|
+
"then": {
|
|
131
|
+
"required": ["type", "sql"],
|
|
132
|
+
"properties": {
|
|
133
|
+
"type": { "const": "duckdb" },
|
|
134
|
+
"sql": {
|
|
135
|
+
"type": "string",
|
|
136
|
+
"minLength": 1,
|
|
137
|
+
"description": "DuckDB SQL escape hatch, for bespoke shapes only. The runner shells out to the `duckdb` CLI (no npm dep) and substitutes the literal token {{PREFIX}} with the run's data prefix path (e.g. data/verify-<name>), so globs read read_json_auto('{{PREFIX}}-EVENTS*.json'). Result rows feed select/expect like emulator rows. Disk mode only — skipped (with a warning) under --in-memory."
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"if": {
|
|
144
|
+
"properties": { "type": { "const": "warehouse" } }
|
|
145
|
+
},
|
|
146
|
+
"then": {
|
|
147
|
+
"required": ["type", "table"],
|
|
148
|
+
"properties": {
|
|
149
|
+
"type": { "const": "warehouse" },
|
|
150
|
+
"table": { "type": "string", "minLength": 1 }
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"if": {
|
|
156
|
+
"properties": { "type": { "const": "warehouse-stats" } }
|
|
157
|
+
},
|
|
158
|
+
"then": {
|
|
159
|
+
"required": ["type", "table"],
|
|
160
|
+
"properties": {
|
|
161
|
+
"type": { "const": "warehouse-stats" },
|
|
162
|
+
"table": { "type": "string", "minLength": 1 }
|
|
163
|
+
}
|
|
137
164
|
}
|
|
138
165
|
}
|
|
139
|
-
|
|
140
|
-
"
|
|
141
|
-
"description": "Anything other than 'duckdb' is passed byte-compatible to emulateBreakdown / verifyDungeon (frequencyByFrequency, funnelFrequency, aggregatePerUser, timeToConvert, attributedBy, sessionMetrics, retention, distinctCount, eventBreakdown, uniques, lifecycle, topPaths)."
|
|
142
|
-
}
|
|
166
|
+
],
|
|
167
|
+
"description": "Anything other than 'duckdb', 'warehouse', or 'warehouse-stats' is passed byte-compatible to emulateBreakdown / verifyDungeon (frequencyByFrequency, funnelFrequency, aggregatePerUser, timeToConvert, attributedBy, sessionMetrics, retention, distinctCount, eventBreakdown, uniques, lifecycle, topPaths)."
|
|
143
168
|
},
|
|
144
169
|
"expect": {
|
|
145
170
|
"type": "object",
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Funnel `conditions` matching (v1.7.0 — P0-1).
|
|
3
|
+
*
|
|
4
|
+
* A funnel with `conditions` is offered only to users whose profile satisfies
|
|
5
|
+
* every key (AND across keys). Each value is either:
|
|
6
|
+
*
|
|
7
|
+
* - a scalar → strict equality (`profile[key] === value`), unchanged since 1.3
|
|
8
|
+
* - an operator map → every operator in the map must hold (AND within a key)
|
|
9
|
+
*
|
|
10
|
+
* Operators: `eq`, `neq`, `in`, `nin`, `gt`, `gte`, `lt`, `lte`.
|
|
11
|
+
* `in` / `nin` take arrays; the rest take scalars. `gt`/`gte`/`lt`/`lte` use
|
|
12
|
+
* JavaScript's `<` / `>` so numbers and ISO date strings both compare.
|
|
13
|
+
*
|
|
14
|
+
* A missing profile key never satisfies `eq`, `in`, or an ordering operator;
|
|
15
|
+
* it does satisfy `neq` and `nin` (the value is not the excluded one).
|
|
16
|
+
*
|
|
17
|
+
* The validator (`validateFunnelConditions`) rejects functions, bare arrays,
|
|
18
|
+
* unknown operators, and `in`/`nin` without an array — those shapes used to
|
|
19
|
+
* silently never match and hand the author an empty funnel.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export const CONDITION_OPERATORS = Object.freeze(['eq', 'neq', 'in', 'nin', 'gt', 'gte', 'lt', 'lte']);
|
|
23
|
+
const OPERATOR_SET = new Set(CONDITION_OPERATORS);
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* True when `value` is an operator map (plain object, not array/Date).
|
|
27
|
+
* @param {unknown} value
|
|
28
|
+
* @returns {value is Record<string, unknown>}
|
|
29
|
+
*/
|
|
30
|
+
export function isOperatorMap(value) {
|
|
31
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {Record<string, unknown>} profile
|
|
36
|
+
* @param {Record<string, unknown>} conditions
|
|
37
|
+
* @returns {boolean}
|
|
38
|
+
*/
|
|
39
|
+
export function matchConditions(profile, conditions) {
|
|
40
|
+
if (!conditions) return true;
|
|
41
|
+
for (const [key, cond] of Object.entries(conditions)) {
|
|
42
|
+
const actual = profile ? profile[key] : undefined;
|
|
43
|
+
if (!isOperatorMap(cond)) {
|
|
44
|
+
if (actual !== cond) return false;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
for (const [op, expected] of Object.entries(cond)) {
|
|
48
|
+
if (!OPERATOR_SET.has(op)) return false; // validator throws earlier; defensive
|
|
49
|
+
switch (op) {
|
|
50
|
+
case 'eq': if (actual !== expected) return false; break;
|
|
51
|
+
case 'neq': if (actual === expected) return false; break;
|
|
52
|
+
case 'in': if (!Array.isArray(expected) || !expected.includes(actual)) return false; break;
|
|
53
|
+
case 'nin': if (Array.isArray(expected) && expected.includes(actual)) return false; break;
|
|
54
|
+
case 'gt': if (!(/** @type {any} */ (actual) > /** @type {any} */ (expected))) return false; break;
|
|
55
|
+
case 'gte': if (!(/** @type {any} */ (actual) >= /** @type {any} */ (expected))) return false; break;
|
|
56
|
+
case 'lt': if (!(/** @type {any} */ (actual) < /** @type {any} */ (expected))) return false; break;
|
|
57
|
+
case 'lte': if (!(/** @type {any} */ (actual) <= /** @type {any} */ (expected))) return false; break;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
@@ -48,9 +48,19 @@ export function evaluateFunctionCall(funcCall) {
|
|
|
48
48
|
|
|
49
49
|
const { functionName, args, body } = funcCall;
|
|
50
50
|
|
|
51
|
-
// Special handling for arrow functions
|
|
51
|
+
// Special handling for arrow functions.
|
|
52
|
+
// v1.7.0 (P1-1): emit `(ctx) => body` so serialized dungeons can read the
|
|
53
|
+
// value context (`ctx.profile`, `ctx.event`, `ctx.time`, `ctx.config`).
|
|
54
|
+
// Bodies that ignore `ctx` behave exactly as before.
|
|
52
55
|
if (functionName === 'arrow') {
|
|
53
|
-
|
|
56
|
+
// `dungeon-to-json` stores a whole function source as the body
|
|
57
|
+
// (`(ctx) => ctx.profile.plan`, `function () { … }`). Emit it as-is so the
|
|
58
|
+
// revived value is that function, not a thunk returning it.
|
|
59
|
+
const trimmed = String(body).trim();
|
|
60
|
+
const isWholeFunction = /^(async\s+)?function\b/.test(trimmed)
|
|
61
|
+
|| /^(async\s*)?\([^)]*\)\s*=>/.test(trimmed)
|
|
62
|
+
|| /^(async\s+)?[A-Za-z_$][\w$]*\s*=>/.test(trimmed);
|
|
63
|
+
return isWholeFunction ? `(${trimmed})` : `(ctx) => ${body}`;
|
|
54
64
|
}
|
|
55
65
|
|
|
56
66
|
// Handle chance.* functions
|