madden-fantasy-extractor 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -0
- package/package.json +25 -0
- package/src/cli.js +128 -0
- package/src/extract/clock.js +75 -0
- package/src/extract/cumulative-stats.js +35 -0
- package/src/extract/depth-chart.js +56 -0
- package/src/extract/games.js +78 -0
- package/src/extract/payload.js +125 -0
- package/src/extract/player-game-stats.js +86 -0
- package/src/extract/player-week-status.js +189 -0
- package/src/extract/players.js +74 -0
- package/src/extract/team-game-stats.js +153 -0
- package/src/extract/teams.js +41 -0
- package/src/extract/transactions.js +105 -0
- package/src/index.js +4 -0
- package/src/lib/franchise-table.js +88 -0
- package/src/lib/identity.js +58 -0
- package/src/lib/incremental.js +90 -0
- package/src/lib/signing.js +48 -0
- package/src/lib/stat-categories.js +68 -0
- package/src/lib/state.js +30 -0
- package/test/helpers/fake-franchise.js +36 -0
- package/test/identity.test.js +61 -0
- package/test/incremental.test.js +147 -0
- package/test/player-game-stats.test.js +85 -0
- package/test/player-week-status.test.js +173 -0
- package/test/state.test.js +54 -0
- package/test/team-game-stats.test.js +119 -0
- package/test/transactions.test.js +118 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// Extracts sim_transaction from PlayerTransactionHistoryEntry — findings.md §7: this
|
|
2
|
+
// table gives the REASON for every move directly, strictly better than diffing two
|
|
3
|
+
// saves. Classification follows the transition table documented there, PLUS ps_sign
|
|
4
|
+
// (added after real-save validation — see db/schema.sql's comment on classified_type):
|
|
5
|
+
//
|
|
6
|
+
// Trade | OldTeam != NewTeam (both real), Signed->Signed
|
|
7
|
+
// Sign (FA) | null -> real team, FreeAgent->Signed
|
|
8
|
+
// Cut | real team -> null, Signed->FreeAgent
|
|
9
|
+
// PS demote | same team, Signed->PracticeSquad
|
|
10
|
+
// PS poach | diff team, PracticeSquad->Signed
|
|
11
|
+
// PS sign | null -> real team, FreeAgent->PracticeSquad (NOT in findings.md's
|
|
12
|
+
// | original table — turned out to be the SINGLE MOST COMMON transaction
|
|
13
|
+
// | type, ~41% of all rows in a sampled save: a street free agent signed
|
|
14
|
+
// | straight to the practice squad, distinct from ps_demote which moves an
|
|
15
|
+
// | already-Signed player down within the same team)
|
|
16
|
+
//
|
|
17
|
+
// Capacity is finite (742, findings §7) — a rolling window, not full history. Every
|
|
18
|
+
// import must archive what it sees; dedupe_key (built from content, since
|
|
19
|
+
// TransactionId is only unique WITHIN one save's current window, not stable across
|
|
20
|
+
// saves/re-imports) is what makes repeated archiving idempotent.
|
|
21
|
+
//
|
|
22
|
+
// CAVEAT (schema.sql): SeasonStage here is PlayerTransactionHistoryEntry's own enum
|
|
23
|
+
// (PreSeason/NFLSeason) — a DIFFERENT vocabulary from SeasonGame.SeasonWeekType
|
|
24
|
+
// (RegularSeason/WildcardPlayoff/...). Never cross-compare the two; both are carried
|
|
25
|
+
// through unchanged below rather than coerced into franchise_week's week_type.
|
|
26
|
+
|
|
27
|
+
import { getTableByNameAndCapacity, liveRecords, resolveFieldRef } from '../lib/franchise-table.js';
|
|
28
|
+
import { createHash } from 'node:crypto';
|
|
29
|
+
|
|
30
|
+
export async function extractTransactions(franchise, { indexTeamsByTeamIndex }) {
|
|
31
|
+
const table = await getTableByNameAndCapacity(franchise, 'PlayerTransactionHistoryEntry');
|
|
32
|
+
const records = liveRecords(table);
|
|
33
|
+
|
|
34
|
+
const rows = [];
|
|
35
|
+
for (const r of records) {
|
|
36
|
+
const playerRecord = await resolveFieldRef(franchise, r, 'Player');
|
|
37
|
+
if (!playerRecord) continue; // unresolvable player ref — skip rather than emit a dangling transaction
|
|
38
|
+
|
|
39
|
+
const oldTeamRecord = await resolveFieldRef(franchise, r, 'OldTeam');
|
|
40
|
+
const newTeamRecord = await resolveFieldRef(franchise, r, 'NewTeam');
|
|
41
|
+
const oldTeam = oldTeamRecord ? indexTeamsByTeamIndex.get(oldTeamRecord.TeamIndex) : null;
|
|
42
|
+
const newTeam = newTeamRecord ? indexTeamsByTeamIndex.get(newTeamRecord.TeamIndex) : null;
|
|
43
|
+
|
|
44
|
+
const classifiedType = classify({
|
|
45
|
+
oldExternalTeamId: oldTeam?.externalTeamId ?? null,
|
|
46
|
+
newExternalTeamId: newTeam?.externalTeamId ?? null,
|
|
47
|
+
oldContractStatus: r.OldContractStatus,
|
|
48
|
+
newContractStatus: r.ContractStatus,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
const row = {
|
|
52
|
+
externalPlayerId: playerRecord.PresentationId,
|
|
53
|
+
oldExternalTeamId: oldTeam?.externalTeamId ?? null,
|
|
54
|
+
newExternalTeamId: newTeam?.externalTeamId ?? null,
|
|
55
|
+
oldContractStatus: r.OldContractStatus,
|
|
56
|
+
newContractStatus: r.ContractStatus,
|
|
57
|
+
seasonYear: r.SeasonYear,
|
|
58
|
+
// PlayerTransactionHistoryEntry's own stage enum — do not compare against
|
|
59
|
+
// sim_game's SeasonWeekType (see module header caveat).
|
|
60
|
+
seasonStage: r.SeasonStage,
|
|
61
|
+
seasonWeekIndex: r.SeasonWeek,
|
|
62
|
+
classifiedType,
|
|
63
|
+
contract: {
|
|
64
|
+
length: r.ContractLength,
|
|
65
|
+
totalSalary: r.ContractTotalSalary,
|
|
66
|
+
bonus: r.ContractBonus,
|
|
67
|
+
salary: r.ContractSalary,
|
|
68
|
+
capSavingsThisYear: r.CapSavingsThisYear,
|
|
69
|
+
fifthYearOptionCapHit: r.FifthYearOptionCapHit,
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
row.dedupeKey = computeDedupeKey(row);
|
|
73
|
+
rows.push(row);
|
|
74
|
+
}
|
|
75
|
+
return rows;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function classify({ oldExternalTeamId, newExternalTeamId, oldContractStatus, newContractStatus }) {
|
|
79
|
+
const hadTeam = oldExternalTeamId !== null;
|
|
80
|
+
const hasTeam = newExternalTeamId !== null;
|
|
81
|
+
const sameTeam = hadTeam && hasTeam && oldExternalTeamId === newExternalTeamId;
|
|
82
|
+
const diffTeam = hadTeam && hasTeam && oldExternalTeamId !== newExternalTeamId;
|
|
83
|
+
|
|
84
|
+
if (diffTeam && oldContractStatus === 'Signed' && newContractStatus === 'Signed') return 'trade';
|
|
85
|
+
if (!hadTeam && hasTeam && oldContractStatus === 'FreeAgent' && newContractStatus === 'Signed') return 'signing';
|
|
86
|
+
if (hadTeam && !hasTeam && oldContractStatus === 'Signed' && newContractStatus === 'FreeAgent') return 'cut';
|
|
87
|
+
if (sameTeam && oldContractStatus === 'Signed' && newContractStatus === 'PracticeSquad') return 'ps_demote';
|
|
88
|
+
if (diffTeam && oldContractStatus === 'PracticeSquad' && newContractStatus === 'Signed') return 'ps_poach';
|
|
89
|
+
if (!hadTeam && hasTeam && oldContractStatus === 'FreeAgent' && newContractStatus === 'PracticeSquad') return 'ps_sign';
|
|
90
|
+
return 'other';
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function computeDedupeKey(row) {
|
|
94
|
+
const parts = [
|
|
95
|
+
row.externalPlayerId,
|
|
96
|
+
row.oldExternalTeamId,
|
|
97
|
+
row.newExternalTeamId,
|
|
98
|
+
row.oldContractStatus,
|
|
99
|
+
row.newContractStatus,
|
|
100
|
+
row.seasonYear,
|
|
101
|
+
row.seasonStage,
|
|
102
|
+
row.seasonWeekIndex,
|
|
103
|
+
];
|
|
104
|
+
return createHash('sha256').update(parts.join('|')).digest('hex');
|
|
105
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
// Library entry point. Re-exports the extraction API for consumers that already
|
|
2
|
+
// have an opened madden-franchise Franchise instance and want to run extraction
|
|
3
|
+
// themselves, without going through this repo's CLI (src/cli.js).
|
|
4
|
+
export { buildPayload } from './extract/payload.js';
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// Thin helpers over the madden-franchise table/record API.
|
|
2
|
+
//
|
|
3
|
+
// Core primitive confirmed against a real save (Madden NFL 26, PC):
|
|
4
|
+
// a reference field's `referenceData` is `{ tableId, rowNumber }`, resolved via
|
|
5
|
+
// `franchise.getTableById(tableId).records[rowNumber]`. Every relationship in the
|
|
6
|
+
// save (Player.GameStats, GameOffensiveStats.SeasonGame, SeasonGame.HomeTeam, ...)
|
|
7
|
+
// goes through this same shape.
|
|
8
|
+
|
|
9
|
+
/** Non-empty records only — tables have fixed capacity with scattered empty slots. */
|
|
10
|
+
export function liveRecords(table) {
|
|
11
|
+
return table.records.filter((r) => !r.isEmpty);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Table instances can be duplicated (decoys, differing capacity). Pick by highest capacity. */
|
|
15
|
+
export function pickByCapacity(tables, { largest = true } = {}) {
|
|
16
|
+
const sorted = [...tables].sort((a, b) => {
|
|
17
|
+
const ac = a.header?.recordCapacity ?? 0;
|
|
18
|
+
const bc = b.header?.recordCapacity ?? 0;
|
|
19
|
+
return largest ? bc - ac : ac - bc;
|
|
20
|
+
});
|
|
21
|
+
return sorted[0];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export async function getTableByNameAndCapacity(franchise, name, opts) {
|
|
25
|
+
const matches = franchise.tables.filter((t) => t.name === name);
|
|
26
|
+
if (matches.length === 0) throw new Error(`No table named "${name}" in this save`);
|
|
27
|
+
const table = pickByCapacity(matches, opts);
|
|
28
|
+
await table.readRecords();
|
|
29
|
+
return table;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Resolve a reference field to its target record, reading the target table first if needed.
|
|
34
|
+
*
|
|
35
|
+
* An unfilled slot in a wrapper table (e.g. a bye week in GameStats[]) reads back as a
|
|
36
|
+
* reference with tableId 0 — not a missing/null field, still `isReference: true`. tableId 0
|
|
37
|
+
* is never a real table, so it's the reliable "nothing here" signal; treat it as absent.
|
|
38
|
+
*/
|
|
39
|
+
export async function resolveRef(franchise, field) {
|
|
40
|
+
if (!field || !field.isReference) return null;
|
|
41
|
+
const { tableId, rowNumber } = field.referenceData;
|
|
42
|
+
if (!tableId) return null;
|
|
43
|
+
const table = franchise.getTableById(tableId);
|
|
44
|
+
if (!table) return null;
|
|
45
|
+
if (table.records.length === 0) {
|
|
46
|
+
await table.readRecords();
|
|
47
|
+
}
|
|
48
|
+
const record = table.records[rowNumber];
|
|
49
|
+
if (!record || record.isEmpty) return null;
|
|
50
|
+
return record;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Resolve a named field on a record to its target record (see resolveRef). */
|
|
54
|
+
export async function resolveFieldRef(franchise, record, fieldKey) {
|
|
55
|
+
const field = record.fieldsArray.find((f) => f.key === fieldKey);
|
|
56
|
+
return resolveRef(franchise, field);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Wrapper tables like `GameStats[]` expose a dynamic, non-fixed set of numbered
|
|
61
|
+
* slot fields (GameStats0, GameStats1, ...) each a reference to a real row.
|
|
62
|
+
* Findings: the nominal slot count is NOT a ceiling (confirmed 20+ filled slots
|
|
63
|
+
* through a Super Bowl run) — iterate whatever fields are actually present,
|
|
64
|
+
* never assume indices 0..17.
|
|
65
|
+
*/
|
|
66
|
+
export async function resolveWrapperSlots(franchise, wrapperRecord, slotPrefix) {
|
|
67
|
+
const slotFields = wrapperRecord.fieldsArray
|
|
68
|
+
.filter((f) => f.key.startsWith(slotPrefix) && /^\d+$/.test(f.key.slice(slotPrefix.length)))
|
|
69
|
+
.sort((a, b) => Number(a.key.slice(slotPrefix.length)) - Number(b.key.slice(slotPrefix.length)));
|
|
70
|
+
|
|
71
|
+
const resolved = [];
|
|
72
|
+
for (const field of slotFields) {
|
|
73
|
+
const record = await resolveRef(franchise, field);
|
|
74
|
+
if (record) resolved.push(record);
|
|
75
|
+
}
|
|
76
|
+
return resolved;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Player.GameStats / SeasonStats / CareerStats all point at a `*[]` wrapper row,
|
|
81
|
+
* not directly at a stat row. This resolves player -> wrapper -> every non-null
|
|
82
|
+
* per-game stat row in one call.
|
|
83
|
+
*/
|
|
84
|
+
export async function resolveGameStatRows(franchise, playerRecord) {
|
|
85
|
+
const wrapper = await resolveFieldRef(franchise, playerRecord, 'GameStats');
|
|
86
|
+
if (!wrapper) return [];
|
|
87
|
+
return resolveWrapperSlots(franchise, wrapper, 'GameStats');
|
|
88
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Player identity: decode-then-hash. See docs/extraction-findings.md §1, §11 and the
|
|
2
|
+
// comment block on the `player` table in db/schema.sql — this is the linchpin of the
|
|
3
|
+
// whole model and the one place a subtle bug silently corrupts a franchise's history.
|
|
4
|
+
|
|
5
|
+
import { createHash } from 'node:crypto';
|
|
6
|
+
|
|
7
|
+
// Sentinel round/pick pair for undrafted players (confirmed against a real save:
|
|
8
|
+
// round=63, pick=511 for every UDFA sampled, including Signed/FreeAgent/PracticeSquad).
|
|
9
|
+
const UDFA_ROUND = 63;
|
|
10
|
+
const UDFA_PICK = 511;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Player.YearDrafted is RELATIVE to the save's current calendar year, not absolute.
|
|
14
|
+
* realDraftYear = CurrentSeasonYear + YearDrafted + 1
|
|
15
|
+
* Verified against 5 real players' known draft years (findings §11). The raw value
|
|
16
|
+
* shifts by one at every season roll — decode BEFORE hashing, always, or identity_key
|
|
17
|
+
* rotates annually and orphans the franchise's history.
|
|
18
|
+
*/
|
|
19
|
+
export function decodeDraftYear(yearDraftedRaw, currentSeasonYear) {
|
|
20
|
+
return currentSeasonYear + yearDraftedRaw + 1;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function isUdfa(draftRound, draftPick) {
|
|
24
|
+
return draftRound === UDFA_ROUND && draftPick === UDFA_PICK;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Deterministic hash of the identity bundle. Bundle composition depends on how the
|
|
29
|
+
* player entered the league:
|
|
30
|
+
* drafted -> (draft_year, draft_round, draft_pick) is globally unique alone; name is
|
|
31
|
+
* belt-and-braces. Bulletproof (findings §1).
|
|
32
|
+
* udfa -> round/pick are sentinels, so the bundle degrades to name + draft_year.
|
|
33
|
+
* Two generated UDFAs sharing a name in one class could collide — rare,
|
|
34
|
+
* tracked via identity_basis for monitoring (schema.sql migration note 6).
|
|
35
|
+
*/
|
|
36
|
+
export function computeIdentityKey({ firstName, lastName, draftYear, draftRound, draftPick }) {
|
|
37
|
+
const basis = isUdfa(draftRound, draftPick) ? 'udfa' : 'drafted';
|
|
38
|
+
const parts =
|
|
39
|
+
basis === 'drafted'
|
|
40
|
+
? [firstName, lastName, draftYear, draftRound, draftPick]
|
|
41
|
+
: [firstName, lastName, draftYear];
|
|
42
|
+
|
|
43
|
+
const hash = createHash('sha256').update(parts.join('|')).digest('hex');
|
|
44
|
+
return { identityKey: hash, identityBasis: basis };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Statuses that are real, ingestible player rows. Everything else (Deleted, Retired,
|
|
49
|
+
* None, Created) is tombstone/template junk — findings §1 caveat A: every raw
|
|
50
|
+
* PresentationId collision in the save pairs a Deleted/Retired tombstone with a real
|
|
51
|
+
* player, all parked at TeamIndex 32. Confirmed 'None'/'Created' are blank-name
|
|
52
|
+
* template rows at TeamIndex 32 too. Drop all four at ingest (schema.sql §12 rule 1).
|
|
53
|
+
*/
|
|
54
|
+
const INGESTIBLE_CONTRACT_STATUSES = new Set(['Signed', 'PracticeSquad', 'FreeAgent', 'Draft']);
|
|
55
|
+
|
|
56
|
+
export function isIngestiblePlayer(record) {
|
|
57
|
+
return INGESTIBLE_CONTRACT_STATUSES.has(record.ContractStatus);
|
|
58
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Pure logic for incremental (weekly) capture: which games belong in THIS run's
|
|
2
|
+
// output, and which games get recorded as "confirmed captured" afterward.
|
|
3
|
+
//
|
|
4
|
+
// Two different questions, two different gates — do not collapse them:
|
|
5
|
+
//
|
|
6
|
+
// selectGamesToCapture: is this game DECIDED and not already in the watermark?
|
|
7
|
+
// Drives what's included in games / fed into playerGameStats+teamGameStats
|
|
8
|
+
// extraction this run.
|
|
9
|
+
//
|
|
10
|
+
// unionCapturedKeys: did this game actually produce player-level stats THIS run?
|
|
11
|
+
// Drives what gets ADDED to the persisted watermark.
|
|
12
|
+
//
|
|
13
|
+
// These must stay separate. Verified directly against a real save (CAREER-WEEK18):
|
|
14
|
+
// SeasonGame shows a final score AND TeamStats is already fully resolved for every
|
|
15
|
+
// week-17 game (14 of 14), a full week before any player's GameStats[] wrapper gains
|
|
16
|
+
// a slot for that same week (docs/extraction-findings.md §3.5 — the clock/stats lag).
|
|
17
|
+
// So "decided" (used for selection) and "player stats actually landed" (required
|
|
18
|
+
// before marking captured) are genuinely different moments. If a freshly-decided
|
|
19
|
+
// game were marked captured immediately, its player_game_stat rows would be
|
|
20
|
+
// permanently skipped once they do land — selectGamesToCapture never re-examines a
|
|
21
|
+
// game once its key is in the watermark. Team-level stats have no such lag (per the
|
|
22
|
+
// same check), so gating on player-stat presence is a strict superset check: by the
|
|
23
|
+
// time player stats exist, team stats are already guaranteed to.
|
|
24
|
+
//
|
|
25
|
+
// gameKey() (games.js) is deliberately season-agnostic ("RegularSeason:0:3:25") —
|
|
26
|
+
// sufficient within a single extraction run, but persisted indefinitely in a state
|
|
27
|
+
// file it can collide: the same two teams can play the same week-type/index matchup
|
|
28
|
+
// in two different franchise seasons. seasonCaptureKey() prefixes clock.seasonIndex
|
|
29
|
+
// to disambiguate — internal to state tracking only; the payload's own gameKey
|
|
30
|
+
// format is untouched.
|
|
31
|
+
|
|
32
|
+
import { gameKey, isDecidedGame } from '../extract/games.js';
|
|
33
|
+
|
|
34
|
+
export function seasonCaptureKey(seasonIndex, game) {
|
|
35
|
+
return `${seasonIndex}:${gameKey(game)}`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* fullMode=true -> every game the save reports (current/legacy behavior, unfiltered;
|
|
40
|
+
* used for first run, --full/--backfill, and recovering a missed week).
|
|
41
|
+
* fullMode=false -> only DECIDED games not already in alreadyCapturedKeys.
|
|
42
|
+
*/
|
|
43
|
+
export function selectGamesToCapture(games, { fullMode, seasonIndex, alreadyCapturedKeys }) {
|
|
44
|
+
if (fullMode) return games;
|
|
45
|
+
return games.filter(
|
|
46
|
+
(g) => isDecidedGame(g) && !alreadyCapturedKeys.has(seasonCaptureKey(seasonIndex, g)),
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** New watermark set to persist after a successful run: previous keys plus every
|
|
51
|
+
* game in `readyGames` (the subset of this run's included games that actually
|
|
52
|
+
* produced at least one player_game_stat row — see module header). */
|
|
53
|
+
export function unionCapturedKeys(alreadyCapturedKeys, readyGames, seasonIndex) {
|
|
54
|
+
const next = new Set(alreadyCapturedKeys);
|
|
55
|
+
for (const g of readyGames) next.add(seasonCaptureKey(seasonIndex, g));
|
|
56
|
+
return next;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Of the games included this run, which ones are safe to add to the watermark.
|
|
61
|
+
*
|
|
62
|
+
* Normally that's "produced at least one player_game_stat row this run" (see module
|
|
63
|
+
* header on the player-stat lag). PreSeason is a carve-out: confirmed exhaustively
|
|
64
|
+
* against a real save (1,644 players, 17,755 GameStats rows walked) that ZERO are
|
|
65
|
+
* linked to a PreSeason game — preseason box scores structurally never populate
|
|
66
|
+
* player-level GameStats in this data model, only the schedule/score. Gating a
|
|
67
|
+
* PreSeason game's watermark on "produced a player_game_stat row" would mean it can
|
|
68
|
+
* NEVER satisfy the condition, so it would be re-selected by selectGamesToCapture on
|
|
69
|
+
* every single incremental run forever. For PreSeason specifically, "decided" is
|
|
70
|
+
* both the correct and the only available readiness signal.
|
|
71
|
+
*/
|
|
72
|
+
export function selectReadyGames(gamesToCapture, playerGameStats) {
|
|
73
|
+
const producedGameKeys = new Set(playerGameStats.map((row) => row.gameKey));
|
|
74
|
+
return gamesToCapture.filter((g) =>
|
|
75
|
+
g.seasonWeekType === 'PreSeason' ? isDecidedGame(g) : producedGameKeys.has(gameKey(g)),
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* The complement of selectReadyGames: games included in this run's output that have
|
|
81
|
+
* NOT yet produced player stats — still sitting in the retry queue (TeamStats/schedule
|
|
82
|
+
* resolved, player_game_stat not yet). Exposed in the payload (payload.js) as
|
|
83
|
+
* pendingPlayerStatGameKeys so a consumer — or a human eyeballing the output — can see
|
|
84
|
+
* directly which games, if any, are still outstanding, rather than having to infer it
|
|
85
|
+
* by diffing games against playerGameStats themselves.
|
|
86
|
+
*/
|
|
87
|
+
export function selectPendingGames(gamesToCapture, playerGameStats) {
|
|
88
|
+
const readyKeys = new Set(selectReadyGames(gamesToCapture, playerGameStats).map((g) => gameKey(g)));
|
|
89
|
+
return gamesToCapture.filter((g) => !readyKeys.has(gameKey(g)));
|
|
90
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// Ed25519 signing for the extractor's identity. franchise.signing_public_key (see
|
|
2
|
+
// db/schema.sql) is what an ingest server checks stat_import.signature against — it's
|
|
3
|
+
// what lets a stranger trust data pushed for a public franchise (README "Design
|
|
4
|
+
// principles"). Node's built-in crypto supports Ed25519 directly; no dependency needed.
|
|
5
|
+
|
|
6
|
+
import { generateKeyPairSync, sign, verify, createPublicKey } from 'node:crypto';
|
|
7
|
+
import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
|
|
8
|
+
import { dirname } from 'node:path';
|
|
9
|
+
|
|
10
|
+
export function generateKeypair() {
|
|
11
|
+
const { publicKey, privateKey } = generateKeyPairSync('ed25519');
|
|
12
|
+
return {
|
|
13
|
+
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }).toString(),
|
|
14
|
+
privateKeyPem: privateKey.export({ type: 'pkcs8', format: 'pem' }).toString(),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function saveKeypair(keyPath, { publicKeyPem, privateKeyPem }) {
|
|
19
|
+
mkdirSync(dirname(keyPath), { recursive: true });
|
|
20
|
+
writeFileSync(`${keyPath}.private.pem`, privateKeyPem, { mode: 0o600 });
|
|
21
|
+
writeFileSync(`${keyPath}.public.pem`, publicKeyPem, { mode: 0o644 });
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function loadOrCreateKeypair(keyPath) {
|
|
25
|
+
const privatePath = `${keyPath}.private.pem`;
|
|
26
|
+
const publicPath = `${keyPath}.public.pem`;
|
|
27
|
+
|
|
28
|
+
if (existsSync(privatePath) && existsSync(publicPath)) {
|
|
29
|
+
return {
|
|
30
|
+
privateKeyPem: readFileSync(privatePath, 'utf8'),
|
|
31
|
+
publicKeyPem: readFileSync(publicPath, 'utf8'),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const keypair = generateKeypair();
|
|
36
|
+
saveKeypair(keyPath, keypair);
|
|
37
|
+
return keypair;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function signPayload(payloadBytes, privateKeyPem) {
|
|
41
|
+
const signature = sign(null, payloadBytes, privateKeyPem);
|
|
42
|
+
return signature.toString('base64');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function verifySignature(payloadBytes, signatureBase64, publicKeyPem) {
|
|
46
|
+
const publicKey = createPublicKey(publicKeyPem);
|
|
47
|
+
return verify(null, payloadBytes, publicKey, Buffer.from(signatureBase64, 'base64'));
|
|
48
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Mirrors the stat_category seed data in db/schema.sql exactly (source_field values
|
|
2
|
+
// are the confirmed Madden 26 fields from docs/extraction-findings.md §2/§8). This is
|
|
3
|
+
// the single source of truth mapping a raw Madden stat field to a stat_category.key —
|
|
4
|
+
// keep it in sync with the seed INSERT if either changes.
|
|
5
|
+
|
|
6
|
+
// player-scope categories, keyed by the raw field name as it appears on
|
|
7
|
+
// GameOffensiveStats / GameDefensiveStats / GameKickingStats / Game*KPReturnStats.
|
|
8
|
+
export const PLAYER_STAT_FIELD_MAP = {
|
|
9
|
+
// Passing
|
|
10
|
+
PASSCOMPLETED: 'pass_completions',
|
|
11
|
+
PASSATTEMPTS: 'pass_attempts',
|
|
12
|
+
PASSYARDS: 'pass_yards',
|
|
13
|
+
PASSTDS: 'pass_tds',
|
|
14
|
+
PASSINTS: 'pass_ints',
|
|
15
|
+
PASSSACKED: 'pass_sacked',
|
|
16
|
+
// Rushing
|
|
17
|
+
RUSHATTEMPTS: 'rush_attempts',
|
|
18
|
+
RUSHYARDS: 'rush_yards',
|
|
19
|
+
RUSHTDS: 'rush_tds',
|
|
20
|
+
RUSHFUMBLES: 'fumbles_total',
|
|
21
|
+
// Receiving
|
|
22
|
+
RECEIVECATCHES: 'rec_catches',
|
|
23
|
+
RECEIVEYARDS: 'rec_yards',
|
|
24
|
+
RECEIVETDS: 'rec_tds',
|
|
25
|
+
RECEIVEDROPS: 'rec_drops',
|
|
26
|
+
// Kicking (targets excluded — derived_diff, handled by the cumulative-stat extractor)
|
|
27
|
+
KICKEPMADE: 'xp_made',
|
|
28
|
+
KICKEPATTEMPTS: 'xp_attempts',
|
|
29
|
+
KICKFGMADE29ORLESS: 'fg_made_0_29',
|
|
30
|
+
KICKFGMADE30TO39: 'fg_made_30_39',
|
|
31
|
+
KICKFGMADE40TO49: 'fg_made_40_49',
|
|
32
|
+
KICKFGMADE50ORMORE: 'fg_made_50_plus',
|
|
33
|
+
KICKFGATTEMPTS: 'fg_attempts',
|
|
34
|
+
// Punting
|
|
35
|
+
PUNTATTEMPTS: 'punts',
|
|
36
|
+
PUNTYARDS: 'punt_yards',
|
|
37
|
+
PUNTIN20: 'punt_in_20',
|
|
38
|
+
// Individual defense
|
|
39
|
+
DEFTACKLES: 'def_tackles',
|
|
40
|
+
ASSDEFTACKLES: 'def_assists',
|
|
41
|
+
DLINESACKS: 'def_sacks',
|
|
42
|
+
DEFTACKLESFORLOSS: 'def_tfl',
|
|
43
|
+
DSECINTS: 'def_ints',
|
|
44
|
+
DSECINTTDS: 'def_int_tds',
|
|
45
|
+
DEFPASSDEFLECTIONS: 'def_pass_deflections',
|
|
46
|
+
DLINEFORCEDFUMBLES: 'def_forced_fumbles',
|
|
47
|
+
DLINEFUMBLERECOVERIES: 'def_fumble_recoveries',
|
|
48
|
+
DLINEFUMBLETDS: 'def_fumble_tds',
|
|
49
|
+
DLINESAFETIES: 'def_safeties',
|
|
50
|
+
// Returns
|
|
51
|
+
KRETYARDS: 'kick_return_yards',
|
|
52
|
+
KRETTDS: 'kick_return_tds',
|
|
53
|
+
PRETYARDS: 'punt_return_yards',
|
|
54
|
+
PRETTDS: 'punt_return_tds',
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
// team-scope categories, keyed by TeamStats raw field name.
|
|
58
|
+
export const TEAM_STAT_FIELD_MAP = {
|
|
59
|
+
SACKS: 'dst_sacks',
|
|
60
|
+
TAKEAWAYS: 'dst_takeaways',
|
|
61
|
+
TWOPOINTCONVMADE: 'two_point_conversions',
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
// Every stat_category.key this extractor can produce, in insertion order matching
|
|
65
|
+
// the schema seed — used to validate the extractor's output covers what the schema
|
|
66
|
+
// declares scoring-relevant.
|
|
67
|
+
export const ALL_PLAYER_STAT_KEYS = Object.values(PLAYER_STAT_FIELD_MAP);
|
|
68
|
+
export const ALL_TEAM_STAT_KEYS = Object.values(TEAM_STAT_FIELD_MAP);
|
package/src/lib/state.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Local, gitignored state tracking which games have already been captured (and
|
|
2
|
+
// presumably uploaded), so a normal run only emits what's new since last time.
|
|
3
|
+
// See src/lib/incremental.js for the selection/watermark logic that consumes this.
|
|
4
|
+
//
|
|
5
|
+
// "Confirmed uploaded" currently means "the CLI successfully wrote its output file" —
|
|
6
|
+
// there's no ingest/upload endpoint yet (see extractor/README.md). Once one exists,
|
|
7
|
+
// this is the natural place to instead gate the state update on a real upload
|
|
8
|
+
// acknowledgment rather than a successful local write.
|
|
9
|
+
|
|
10
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
|
|
11
|
+
import { dirname } from 'node:path';
|
|
12
|
+
|
|
13
|
+
export function loadState(statePath) {
|
|
14
|
+
if (!existsSync(statePath)) return null;
|
|
15
|
+
const raw = JSON.parse(readFileSync(statePath, 'utf8'));
|
|
16
|
+
return {
|
|
17
|
+
capturedGameKeys: new Set(raw.capturedGameKeys ?? []),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function saveState(statePath, { capturedGameKeys, toolVersion, savePath }) {
|
|
22
|
+
mkdirSync(dirname(statePath), { recursive: true });
|
|
23
|
+
const serialized = {
|
|
24
|
+
capturedGameKeys: [...capturedGameKeys].sort(),
|
|
25
|
+
updatedAt: new Date().toISOString(),
|
|
26
|
+
lastToolVersion: toolVersion,
|
|
27
|
+
lastSavePath: savePath,
|
|
28
|
+
};
|
|
29
|
+
writeFileSync(statePath, JSON.stringify(serialized, null, 2));
|
|
30
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Minimal fakes matching the madden-franchise API surface this extractor actually
|
|
2
|
+
// uses (table.records, table.readRecords(), record.fieldsArray, field.referenceData,
|
|
3
|
+
// franchise.getTableById). Exercises the REAL resolveRef/resolveFieldRef code path
|
|
4
|
+
// rather than reimplementing it, since that resolution logic is where this
|
|
5
|
+
// extractor's correctness lives.
|
|
6
|
+
|
|
7
|
+
let nextTableId = 1;
|
|
8
|
+
|
|
9
|
+
export function fakeTable(name, records) {
|
|
10
|
+
return {
|
|
11
|
+
name,
|
|
12
|
+
_id: nextTableId++,
|
|
13
|
+
header: { recordCapacity: records.length },
|
|
14
|
+
records,
|
|
15
|
+
async readRecords() {},
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** A record with plain fields plus reference fields (key -> {tableId, rowNumber}). */
|
|
20
|
+
export function fakeRecord(fields, refs = {}) {
|
|
21
|
+
const fieldsArray = [
|
|
22
|
+
...Object.keys(fields).map((key) => ({ key })),
|
|
23
|
+
...Object.keys(refs).map((key) => ({ key, isReference: true, referenceData: refs[key] })),
|
|
24
|
+
];
|
|
25
|
+
return { ...fields, isEmpty: false, fieldsArray };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Wires a list of fakeTable() instances into a minimal franchise-shaped object. */
|
|
29
|
+
export function fakeFranchise(tables) {
|
|
30
|
+
return {
|
|
31
|
+
tables,
|
|
32
|
+
getTableById(id) {
|
|
33
|
+
return tables.find((t) => t._id === id);
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { decodeDraftYear, computeIdentityKey, isIngestiblePlayer } from '../src/lib/identity.js';
|
|
4
|
+
|
|
5
|
+
test('decodeDraftYear matches the 5 verified players from extraction-findings.md §11', () => {
|
|
6
|
+
// realDraftYear = CurrentSeasonYear + YearDrafted + 1, CurrentSeasonYear=2025
|
|
7
|
+
assert.equal(decodeDraftYear(-8, 2025), 2018); // Josh Allen
|
|
8
|
+
assert.equal(decodeDraftYear(-9, 2025), 2017); // Patrick Mahomes
|
|
9
|
+
assert.equal(decodeDraftYear(-8, 2025), 2018); // Lamar Jackson
|
|
10
|
+
assert.equal(decodeDraftYear(-6, 2025), 2020); // Joe Burrow / Justin Herbert
|
|
11
|
+
assert.equal(decodeDraftYear(-3, 2025), 2023); // C.J. Stroud
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test('decodeDraftYear shifts correctly across a season roll', () => {
|
|
15
|
+
// findings.md: "the raw value shifts by one at every season roll" — same player,
|
|
16
|
+
// decoded year must stay identical even though YearDrafted itself changes.
|
|
17
|
+
const decodedIn2025 = decodeDraftYear(-8, 2025);
|
|
18
|
+
const decodedIn2026 = decodeDraftYear(-9, 2026);
|
|
19
|
+
assert.equal(decodedIn2025, decodedIn2026);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test('computeIdentityKey is deterministic and stable for the same inputs', () => {
|
|
23
|
+
const input = { firstName: 'Josh', lastName: 'Allen', draftYear: 2018, draftRound: 1, draftPick: 7 };
|
|
24
|
+
const a = computeIdentityKey(input);
|
|
25
|
+
const b = computeIdentityKey({ ...input });
|
|
26
|
+
assert.equal(a.identityKey, b.identityKey);
|
|
27
|
+
assert.equal(a.identityBasis, 'drafted');
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test('computeIdentityKey classifies the UDFA sentinel (round=63, pick=511) as udfa basis', () => {
|
|
31
|
+
const { identityBasis } = computeIdentityKey({
|
|
32
|
+
firstName: 'Some', lastName: 'Player', draftYear: 2022, draftRound: 63, draftPick: 511,
|
|
33
|
+
});
|
|
34
|
+
assert.equal(identityBasis, 'udfa');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('computeIdentityKey produces different keys for different players', () => {
|
|
38
|
+
const a = computeIdentityKey({ firstName: 'Josh', lastName: 'Allen', draftYear: 2018, draftRound: 1, draftPick: 7 });
|
|
39
|
+
const b = computeIdentityKey({ firstName: 'Lamar', lastName: 'Jackson', draftYear: 2018, draftRound: 1, draftPick: 32 });
|
|
40
|
+
assert.notEqual(a.identityKey, b.identityKey);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('computeIdentityKey UDFA bundle ignores round/pick (both are sentinels)', () => {
|
|
44
|
+
// Two UDFAs with the same name+draftYear collide by design (schema.sql migration
|
|
45
|
+
// note 6) — this documents that behavior rather than treating it as a bug.
|
|
46
|
+
const a = computeIdentityKey({ firstName: 'John', lastName: 'Doe', draftYear: 2024, draftRound: 63, draftPick: 511 });
|
|
47
|
+
const b = computeIdentityKey({ firstName: 'John', lastName: 'Doe', draftYear: 2024, draftRound: 63, draftPick: 511 });
|
|
48
|
+
assert.equal(a.identityKey, b.identityKey);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test('isIngestiblePlayer accepts real roster statuses', () => {
|
|
52
|
+
for (const status of ['Signed', 'PracticeSquad', 'FreeAgent', 'Draft']) {
|
|
53
|
+
assert.equal(isIngestiblePlayer({ ContractStatus: status }), true, status);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('isIngestiblePlayer rejects tombstone/template statuses', () => {
|
|
58
|
+
for (const status of ['Deleted', 'Retired', 'None', 'Created']) {
|
|
59
|
+
assert.equal(isIngestiblePlayer({ ContractStatus: status }), false, status);
|
|
60
|
+
}
|
|
61
|
+
});
|