football-docs 0.1.2 → 0.2.0

package/dist/ingest.js

@@ -1,35 +1,67 @@
 /**
  * Ingest provider documentation into the SQLite FTS5 index.
  *
- * Reads markdown files from docs/providers/{provider}/*.md and chunks them
- * by heading (## or ###), storing each chunk with metadata.
+ * Reads markdown files from docs/{provider}/*.md and chunks them
+ * by heading (## or ###), storing each chunk with provenance metadata.
+ *
+ * Each markdown file can include a YAML frontmatter block with provenance:
+ *   ---
+ *   source_url: https://example.com/docs
+ *   source_type: readthedocs
+ *   upstream_version: 1.8.8
+ *   crawled_at: 2026-03-26T00:00:00Z
+ *   ---
+ *
+ * Files without frontmatter default to source_type: "curated".
  *
  * Usage:
  *   npm run ingest                    # ingest all providers
  *   npm run ingest -- --provider opta # ingest one provider
- *
- * Doc file naming convention:
- *   docs/providers/{provider}/{category}.md
- *
- * Example:
- *   docs/providers/opta/event-types.md
- *   docs/providers/opta/qualifiers.md
- *   docs/providers/opta/coordinate-system.md
- *   docs/providers/statsbomb/events.md
- *   docs/providers/kloppy/data-model.md
  */
-import Database from "better-sqlite3";
-import { resolve, dirname, basename } from "node:path";
+import { existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
+import { basename, dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
+import Database from "better-sqlite3";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const DOCS_DIR = resolve(__dirname, "..", "docs");
 const DB_DIR = resolve(__dirname, "..", "data");
 const DB_PATH = resolve(DB_DIR, "docs.db");
+/** Parse optional YAML frontmatter from a markdown file. */
+export function parseFrontmatter(text) {
+    const defaults = {
+        source_url: null,
+        source_type: "curated",
+        upstream_version: null,
+        crawled_at: null,
+    };
+    if (!text.startsWith("---")) {
+        return { frontmatter: defaults, body: text };
+    }
+    const endIndex = text.indexOf("\n---", 3);
+    if (endIndex === -1) {
+        return { frontmatter: defaults, body: text };
+    }
+    const yamlBlock = text.slice(4, endIndex);
+    const body = text.slice(endIndex + 4).trimStart();
+    // Simple YAML key: value parser (no dependency needed for flat frontmatter)
+    const fm = { ...defaults };
+    for (const line of yamlBlock.split("\n")) {
+        const match = line.match(/^(\w+):\s*(.+)$/);
+        if (match) {
+            const [, key, value] = match;
+            const cleaned = value.replace(/^["']|["']$/g, "").trim();
+            if (key in fm) {
+                fm[key] = cleaned || null;
+            }
+        }
+    }
+    return { frontmatter: fm, body };
+}
 /** Split a markdown file into chunks by ## or ### headings. */
-function chunkMarkdown(text, provider, category) {
+export function chunkMarkdown(text, provider, category) {
+    const { frontmatter, body } = parseFrontmatter(text);
     const chunks = [];
-    const lines = text.split("\n");
+    const lines = body.split("\n");
     let currentTitle = `${provider} - ${category}`;
     let currentLines = [];
     for (const line of lines) {
@@ -37,7 +69,7 @@ function chunkMarkdown(text, provider, category) {
         if (headingMatch && currentLines.length > 0) {
             const content = currentLines.join("\n").trim();
             if (content.length > 20) {
-                chunks.push({ provider, category, title: currentTitle, content });
+                chunks.push({ provider, category, title: currentTitle, content, ...frontmatter });
             }
             currentTitle = headingMatch[2].trim();
             currentLines = [line];
@@ -48,7 +80,7 @@ function chunkMarkdown(text, provider, category) {
     }
     const content = currentLines.join("\n").trim();
     if (content.length > 20) {
-        chunks.push({ provider, category, title: currentTitle, content });
+        chunks.push({ provider, category, title: currentTitle, content, ...frontmatter });
     }
     return chunks;
 }
@@ -61,60 +93,92 @@ function ingestProvider(db, provider) {
     }
     const files = readdirSync(providerDir).filter((f) => f.endsWith(".md"));
     let totalChunks = 0;
-    const insert = db.prepare("INSERT INTO docs (provider, category, title, content) VALUES (?, ?, ?, ?)");
+    const insert = db.prepare(`INSERT INTO docs (provider, category, title, content, source_url, source_type, upstream_version, crawled_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
     for (const file of files) {
         const category = basename(file, ".md");
         const text = readFileSync(resolve(providerDir, file), "utf-8");
         const chunks = chunkMarkdown(text, provider, category);
         for (const chunk of chunks) {
-            insert.run(chunk.provider, chunk.category, chunk.title, chunk.content);
+            insert.run(chunk.provider, chunk.category, chunk.title, chunk.content, chunk.source_url, chunk.source_type, chunk.upstream_version, chunk.crawled_at);
         }
         totalChunks += chunks.length;
-        console.log(`  ${provider}/${file}: ${chunks.length} chunks`);
+        const sourceTag = chunks[0]?.source_type ?? "curated";
+        console.log(`  ${provider}/${file}: ${chunks.length} chunks [${sourceTag}]`);
     }
     return totalChunks;
 }
+export const SCHEMA_SQL = `
+  CREATE TABLE IF NOT EXISTS docs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    provider TEXT NOT NULL,
+    category TEXT NOT NULL,
+    title TEXT NOT NULL,
+    content TEXT NOT NULL,
+    source_url TEXT,
+    source_type TEXT NOT NULL DEFAULT 'curated',
+    upstream_version TEXT,
+    crawled_at TEXT
+  );
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(
+    provider,
+    category,
+    title,
+    content,
+    content='docs',
+    content_rowid='id',
+    tokenize='porter unicode61'
+  );
+
+  CREATE TRIGGER IF NOT EXISTS docs_ai AFTER INSERT ON docs BEGIN
+    INSERT INTO docs_fts(rowid, provider, category, title, content)
+    VALUES (new.id, new.provider, new.category, new.title, new.content);
+  END;
+`;
+/** Ensure tables exist without dropping existing data. */
+function ensureSchema(db) {
+    db.exec(SCHEMA_SQL);
+}
+/** Drop everything and recreate from scratch. */
+function rebuildSchema(db) {
+    db.exec(`
+    DROP TABLE IF EXISTS docs_fts;
+    DROP TABLE IF EXISTS docs;
+  `);
+    db.exec(SCHEMA_SQL);
+}
 function main() {
     const providerArg = process.argv.indexOf("--provider");
     const singleProvider = providerArg >= 0 ? process.argv[providerArg + 1] : undefined;
     mkdirSync(DB_DIR, { recursive: true });
     const db = new Database(DB_PATH);
-    db.exec(`
-    DROP TABLE IF EXISTS docs_fts;
-    DROP TABLE IF EXISTS docs;
-
-    CREATE TABLE docs (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      provider TEXT NOT NULL,
-      category TEXT NOT NULL,
-      title TEXT NOT NULL,
-      content TEXT NOT NULL
-    );
-
-    CREATE VIRTUAL TABLE docs_fts USING fts5(
-      provider,
-      category,
-      title,
-      content,
-      content='docs',
-      content_rowid='id',
-      tokenize='porter unicode61'
-    );
-
-    CREATE TRIGGER docs_ai AFTER INSERT ON docs BEGIN
-      INSERT INTO docs_fts(rowid, provider, category, title, content)
-      VALUES (new.id, new.provider, new.category, new.title, new.content);
-    END;
-  `);
-    console.log("Ingesting provider docs...\n");
+    db.pragma("journal_mode = WAL");
     if (singleProvider) {
+        // Single-provider mode: only rebuild that provider's data
+        // Ensure tables exist (in case DB is fresh)
+        ensureSchema(db);
+        // Atomically delete existing data and re-ingest for this provider
+        const deleteAndReinsert = db.transaction(() => {
+            const rows = db.prepare("SELECT id, provider, category, title, content FROM docs WHERE provider = ?").all(singleProvider);
+            const ftsDelete = db.prepare("INSERT INTO docs_fts(docs_fts, rowid, provider, category, title, content) VALUES('delete', ?, ?, ?, ?, ?)");
+            for (const row of rows) {
+                ftsDelete.run(row.id, row.provider, row.category, row.title, row.content);
+            }
+            db.prepare("DELETE FROM docs WHERE provider = ?").run(singleProvider);
+        });
+        deleteAndReinsert();
+        console.log(`Ingesting ${singleProvider}...\n`);
         const count = ingestProvider(db, singleProvider);
         console.log(`\nDone: ${count} chunks from ${singleProvider}`);
     }
     else {
+        // Full rebuild: drop and recreate everything
+        rebuildSchema(db);
+        console.log("Ingesting provider docs...\n");
         if (!existsSync(DOCS_DIR)) {
             console.log(`No docs directory at ${DOCS_DIR}`);
-            console.log("Create docs/providers/{provider}/*.md files first.");
+            console.log("Create docs/{provider}/*.md files first.");
             db.close();
             return;
         }
@@ -130,4 +194,8 @@ function main() {
     }
     db.close();
 }
-main();
+// Only run when executed directly, not when imported for tests
+const isDirectRun = process.argv[1]?.endsWith("ingest.ts") || process.argv[1]?.endsWith("ingest.js");
+if (isDirectRun) {
+    main();
+}

package/docs/databallpy/data-model.md

@@ -0,0 +1,146 @@
+# DataBallPy Data Model
+
+## Game Object
+
+The `Game` dataclass is the central object, holding tracking data, event data, and match metadata.
+
+**Key attributes**:
+
+| Attribute | Type | Description |
+|---|---|---|
+| `tracking_data` | `TrackingData` | Custom pandas DataFrame subclass |
+| `event_data` | `EventData` | Custom pandas DataFrame subclass |
+| `pitch_dimensions` | `list[float]` | [length, width] in metres (typically [106.0, 68.0]) |
+| `periods` | `pd.DataFrame` | Start/end frames and datetimes for periods 1-5 |
+| `home_team_id` | `int` | |
+| `home_team_name` | `str` | |
+| `home_players` | `pd.DataFrame` | Player info (see PlayersSchema) |
+| `home_score` | `int` | |
+| `home_formation` | `str` | |
+| `away_team_id` | `int` | |
+| `away_team_name` | `str` | |
+| `away_players` | `pd.DataFrame` | |
+| `away_score` | `int` | |
+| `away_formation` | `str` | |
+| `country` | `str \| None` | |
+| `shot_events` | `pd.DataFrame` | Typed shot events |
+| `dribble_events` | `pd.DataFrame` | Typed dribble events |
+| `pass_events` | `pd.DataFrame` | Typed pass events |
+
+**Key methods**: `synchronise_tracking_and_event_data()`, `get_column_ids()`, `get_event()`, `get_frames()`, `get_event_frame()`, `save_game()`, `copy()`.
+
+**Key properties**: `is_synchronised`, `tracking_timestamp_is_precise`, `event_timestamp_is_precise`, `date`, `name`.
+
+## TrackingData
+
+Custom `pd.DataFrame` subclass with extra attributes: `provider: str`, `frame_rate: int|float`.
+
+**Required columns** (validated by Pandera schema):
+
+| Column | Type | Description |
+|---|---|---|
+| `frame` | int (unique) | Frame number |
+| `datetime` | datetime | Timestamp |
+| `ball_x` | float | Ball x position (metres from centre) |
+| `ball_y` | float | Ball y position (metres from centre) |
+| `ball_z` | float | Ball height |
+| `ball_status` | str | `"alive"` or `"dead"` |
+| `team_possession` | str | Which team has possession |
+
+**Player columns** follow the pattern: `home_1_x`, `home_1_y`, `away_3_x`, `away_3_y`, etc. Player IDs map to the `home_players` / `away_players` DataFrames on the Game object.
+
+**Added columns** (from preprocessing methods):
+- `{col_id}_vx`, `{col_id}_vy` -- velocity (from `add_velocity()`)
+- `{col_id}_ax`, `{col_id}_ay` -- acceleration (from `add_acceleration()`)
+- `event_id`, `databallpy_event`, `sync_certainty` -- after synchronization
+
+## EventData
+
+Custom `pd.DataFrame` subclass with extra attribute: `provider: str`.
+
+**Required columns** (validated by Pandera schema):
+
+| Column | Type | Description |
+|---|---|---|
+| `event_id` | int (unique) | Event identifier |
+| `databallpy_event` | str (nullable) | One of: `"pass"`, `"shot"`, `"dribble"`, `"tackle"`, `"own_goal"` |
+| `period_id` | int | Period number |
+| `minutes` | int | Minute of event |
+| `seconds` | int | Second within minute |
+| `player_id` | int | Player identifier |
+| `player_name` | str | |
+| `team_id` | int | |
+| `is_successful` | bool | |
+| `start_x` | float | Range: -60 to 60 |
+| `start_y` | float | Range: -45 to 45 |
+| `datetime` | datetime | Timestamp |
+| `original_event_id` | str | Provider's event ID |
+| `original_event` | str | Provider's event type name |
+
+**Optional columns**: `end_x`, `end_y`, `to_player_id`, `to_player_name`, `event_type_id`, `team_name`.
+
+**Added columns after synchronization**: `tracking_frame`, `sync_certainty`.
+
+## PlayersSchema
+
+| Column | Type | Constraints |
+|---|---|---|
+| `id` | int | Unique |
+| `full_name` | str | Unique |
+| `shirt_num` | int | Unique, 0-100 |
+| `position` | str | goalkeeper / defender / midfielder / forward / unspecified |
+| `start_frame` | int | |
+| `end_frame` | int | |
+| `starter` | bool | |
+
+## Event Classes
+
+All event classes are dataclasses in `databallpy/events/` with strict validation.
+
+### IndividualCloseToBallEvent (base)
+
+Fields: `event_id`, `period_id`, `minutes`, `seconds`, `datetime`, `start_x`, `start_y`, `team_id`, `team_side`, `pitch_size`, `player_id`, `jersey`, `outcome`, `related_event_id`.
+
+### IndividualOnBallEvent (extends base)
+
+Adds: `body_part`, `possession_type`, `set_piece`, plus computed `xt` (expected threat) property.
+
+**body_part values**: `right_foot`, `left_foot`, `foot`, `head`, `other`, `unspecified`.
+
+**set_piece values**: `goal_kick`, `free_kick`, `throw_in`, `corner_kick`, `kick_off`, `penalty`, `no_set_piece`, `unspecified`.
+
+**possession_type values**: `open_play`, `counter_attack`, `corner_kick`, `free_kick`, `throw_in`, `penalty`, `kick_off`, `goal_kick`, `rebound`, `unspecified`.
+
+### ShotEvent
+
+Extends `IndividualOnBallEvent`. Adds: `outcome_str`, `y_target`, `z_target`, `first_touch`, `ball_goal_distance`, `shot_angle`, `xg` (auto-computed).
+
+**outcome_str values**: `goal`, `miss_off_target`, `miss_hit_post`, `miss_on_target`, `blocked`, `own_goal`, `miss`, `unspecified`.
+
+xG model: logistic regression on distance and angle, with separate models for foot/head/free_kick/penalty. Penalty xG = 0.79.
+
+### PassEvent
+
+Extends `IndividualOnBallEvent`. Adds: `outcome_str`, `end_x`, `end_y`, `pass_type`, `receiver_player_id`.
+
+**outcome_str values**: `successful`, `unsuccessful`, `offside`, `results_in_shot`, `assist`, `fair_play`, `unspecified`.
+
+**pass_type values**: `long_ball`, `cross`, `through_ball`, `chipped`, `lay-off`, `lounge`, `flick_on`, `pull_back`, `switch_off_play`, `unspecified`.
+
+### DribbleEvent
+
+Extends `IndividualOnBallEvent`. Adds: `duel_type` (offensive/defensive/unspecified), `with_opponent` (bool).
+
+### TackleEvent
+
+Extends `IndividualCloseToBallEvent`. No additional fields.
+
+## Expected Threat (xT)
+
+Karun Singh model with pre-computed lookup tables for open play, free kicks, and throw-ins. Auto-computed as `xt` property on all `IndividualOnBallEvent` subclasses.
+
+Special values: penalty = 0.797, corner kick = 0.049, goal kick = 0.0, kick off = 0.001.
+
+## Missing Values
+
+`MISSING_INT = -999` is used as a sentinel for missing integers throughout the codebase.

package/docs/databallpy/overview.md

@@ -0,0 +1,71 @@
+# DataBallPy
+
+## Overview
+
+DataBallPy is a Python package for loading, synchronizing, and analyzing soccer event and tracking data. Developed by Alexander Oonk and Daan Grob, licensed under MIT. Current version: 0.7.2.
+
+Its key distinguishing feature is **smart synchronization of tracking and event data** using the Needleman-Wunsch algorithm (borrowed from bioinformatics sequence alignment), which preserves event ordering during alignment. Naive cost-function approaches do not guarantee this.
+
+- **Install**: `pip install databallpy` (optional: `pip install 'databallpy[kloppy]'` for kloppy integration, `pip install 'databallpy[accessible-space]'` for dangerous accessible space)
+- **Python**: 3.10 through 3.14
+- **Key dependencies**: pandas>=2.1, numpy, scipy, matplotlib, pandera (schema validation), beautifulsoup4, pyarrow
+
+## Supported Providers
+
+### Tracking Data
+
+| Provider | `tracking_data_provider` | Precise Timestamps | Notes |
+|---|---|---|---|
+| Tracab | `"tracab"` | Yes | Includes Sportec/DFL format |
+| Metrica | `"metrica"` | Yes | Has open data support |
+| Inmotio | `"inmotio"` | No | |
+| Sportec/DFL | `"dfl"` or `"sportec"` | Yes | Uses Tracab parser internally |
+
+### Event Data
+
+| Provider | `event_data_provider` | Precise Timestamps | Notes |
+|---|---|---|---|
+| Opta | `"opta"` | Yes | F24 (events) + F7 (metadata) XML files |
+| Metrica | `"metrica"` | Yes | Has open data support |
+| Instat | `"instat"` | No | |
+| SciSports | `"scisports"` | No | No separate metadata file needed |
+| Sportec/DFL | `"dfl"` or `"sportec"` | Yes | |
+| StatsBomb | `"statsbomb"` | No | Requires event_match_loc + event_lineup_loc |
+
+### Kloppy Integration (v0.7.0+)
+
+Install with `pip install 'databallpy[kloppy]'` to parse data from any kloppy-supported provider and convert to a DataBallPy `Game` object via `get_game_from_kloppy()`. This dramatically expands provider coverage.
+
+## Coordinate System
+
+Origin at center of pitch. All data is normalised so:
+- Home team always plays left to right (negative x to positive x)
+- Away team always plays right to left
+- x range: roughly -53 to +53 metres (half pitch length)
+- y range: roughly -34 to +34 metres (half pitch width)
+- Validation bounds: ball_x [-62.5, 62.5], ball_y [-45, 45]
+
+**Playing direction options** when retrieving frames:
+- `"team_oriented"` (default): home left-to-right
+- `"possession_oriented"`: coordinates flipped so possessing team always plays left-to-right
+
+## Built-in Open Data
+
+`get_open_game()` provides 7 DFL/Bundesliga matches (2 Bundesliga 1 + 5 Bundesliga 2):
+
+```python
+from databallpy import get_open_game
+game = get_open_game(provider="sportec", game_id="J03WMX")
+```
+
+## Limitations and Caveats
+
+- **Both data streams often needed**: Many features (synchronization, team possession, event frames) require both tracking and event data.
+- **Simple xG/xT models**: Built-in xG uses basic logistic regression on distance and angle. xT uses pre-computed static lookup tables. Useful for quick analysis, not production-grade.
+- **Sync quality depends on data quality**: If ball_status is wrong, timestamps imprecise, or tracking data starts late, sync degrades.
+- **No Second Spectrum native parser**: Must use Kloppy bridge.
+- **Video export requires ffmpeg**: `save_tracking_video` needs ffmpeg installed.
+- **Deprecation in progress (v0.7.x to v0.8.0)**: `Match` class deprecated in favour of `Game`. Standalone feature functions deprecated in favour of methods on `TrackingData`.
+- **Strict coordinate validation**: Pandera schema validation on load can reject data with positions slightly outside expected bounds.
+- **Performance**: Some features iterate frame-by-frame in Python (Voronoi, pressure), which can be slow for full-game tracking data at 25 Hz (~135,000 frames).
+- **Soccer only**: Despite the generic name, no support for other sports.