@the-situation/indexer 0.1.0-alpha.1

package/README.md

@@ -0,0 +1,236 @@
+# @the-situation/indexer
+
+ETL indexer and REST/WebSocket API for The Situation prediction markets. Polls the [Voyager](https://voyager.online) explorer API for on-chain events, decodes them, stores everything in SQLite, and serves it over HTTP and WebSocket.
+
+## Quick Start
+
+```bash
+# From the repo root
+cp packages/indexer/.env.example packages/indexer/.env
+# Fill in VOYAGER_API_KEY and ADMIN_API_KEY
+
+bun run --filter '@the-situation/indexer' dev
+```
+
+The server starts at `http://localhost:3000`. Register a market to begin indexing:
+
+```bash
+curl -X POST http://localhost:3000/admin/markets \
+  -H "Authorization: Bearer <ADMIN_API_KEY>" \
+  -H "Content-Type: application/json" \
+  -d '{"address": "0x02de...", "title": "My Market", "category": "crypto"}'
+```
+
+Events will begin appearing within ~15 seconds.
+
+## Environment Variables
+
+| Variable | Required | Default | Description |
+|----------|----------|---------|-------------|
+| `VOYAGER_API_KEY` | Yes | — | Voyager explorer API key |
+| `ADMIN_API_KEY` | Yes | — | Bearer token for `/admin/*` endpoints |
+| `DB_PATH` | No | `./data/indexer.db` | SQLite database file path |
+| `PORT` | No | `3000` | HTTP server port |
+| `STARKNET_RPC_URL` | No | `https://api.cartridge.gg/x/starknet/sepolia` | Starknet RPC endpoint |
+| `EVENT_POLL_INTERVAL_MS` | No | `15000` | Event polling interval (ms) |
+| `STATE_POLL_INTERVAL_MS` | No | `30000` | Market state refresh interval (ms) |
+| `POSITION_POLL_INTERVAL_MS` | No | `60000` | Position refresh interval (ms) |
+
+## API Reference
+
+### Public Endpoints
+
+#### `GET /health`
+
+Returns service health and basic stats.
+
+```json
+{
+  "status": "ok",
+  "uptime": 3600,
+  "dbStatus": "connected",
+  "marketsCount": 1,
+  "eventsCount": 42,
+  "lastPollAt": null
+}
+```
+
+#### `GET /api/markets`
+
+List all registered markets with their latest on-chain state.
+
+#### `GET /api/markets/:address`
+
+Get a single market by contract address. Returns `404` if not found.
+
+#### `GET /api/markets/:address/events`
+
+Paginated event time series for a market.
+
+| Query Param | Default | Description |
+|-------------|---------|-------------|
+| `page` | `1` | Page number |
+| `pageSize` | `100` | Items per page (max 500) |
+| `from` | — | Unix timestamp lower bound |
+| `to` | — | Unix timestamp upper bound |
+
+```json
+{
+  "data": [
+    {
+      "id": 1,
+      "txHash": "0x...",
+      "blockNumber": 6290091,
+      "timestamp": 1770487426,
+      "eventType": "trade_executed",
+      "trader": "0x...",
+      "mean": 13,
+      "stdDev": 3,
+      "lowerBound": 10,
+      "upperBound": 16,
+      "oldMean": 12,
+      "oldStdDev": 3,
+      "collateralPosted": "91208000000000000"
+    }
+  ],
+  "total": 42,
+  "page": 1,
+  "pageSize": 100
+}
+```
+
+#### `GET /api/markets/:address/traders`
+
+List all discovered traders and their positions for a market.
+
+#### `GET /api/positions/:trader`
+
+All positions held by a trader across markets.
+
+#### `GET /api/rankings`
+
+Global leaderboard sorted by PnL.
+
+| Query Param | Default | Description |
+|-------------|---------|-------------|
+| `limit` | `50` | Max entries (max 200) |
+
+#### `WS /ws`
+
+WebSocket endpoint for real-time event streaming. Receives JSON messages when new events are indexed:
+
+```json
+{
+  "type": "new_events",
+  "marketAddress": "0x...",
+  "data": { "count": 3, "latestBlock": 6395140 }
+}
+```
+
+### Admin Endpoints
+
+All admin endpoints require `Authorization: Bearer <ADMIN_API_KEY>`.
+
+#### `POST /admin/markets`
+
+Register a new market for indexing.
+
+```bash
+curl -X POST http://localhost:3000/admin/markets \
+  -H "Authorization: Bearer $ADMIN_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "address": "0x02de...",
+    "title": "BTC Price Market",
+    "description": "Predict the price of BTC",
+    "category": "crypto",
+    "topics": ["btc", "price"]
+  }'
+```
+
+#### `PUT /admin/markets/:address`
+
+Update market metadata (title, description, category, topics, is_active).
+
+#### `DELETE /admin/markets/:address`
+
+Soft-delete a market (stops indexing, keeps historical data).
+
+## ETL Pipeline
+
+The indexer runs three polling loops:
+
+| Loop | Interval | Description |
+|------|----------|-------------|
+| **Event indexer** | 15s | Fetches events from Voyager, decodes, stores in SQLite, broadcasts to WebSocket |
+| **State refresher** | 30s | Reads current market state from Starknet RPC |
+| **Position refresher** | 60s | Reads positions for discovered traders from Starknet RPC |
+
+Supported event types:
+
+| Event | Encoding | Description |
+|-------|----------|-------------|
+| `TradeExecuted` | SQ128x128 (4-limb) | Current-generation trade events |
+| `MarketInitialized` | SQ128x128 | Market bootstrap event |
+| `TradeMoved` | Q96 | Legacy trade events |
+| `Bootstrapped` | Q96 | Legacy bootstrap events |
+
+Events are deduplicated by `tx_hash` (UNIQUE constraint). The indexer maintains a per-market cursor tracking the last indexed block number.
+
+## Building
+
+```bash
+# Type-check only
+bun run --filter '@the-situation/indexer' typecheck
+
+# Full build (compiles to dist/)
+bun run --filter '@the-situation/indexer' build
+
+# Run tests
+bun run --filter '@the-situation/indexer' test
+```
+
+## Deploying to Fly.io
+
+The indexer is configured for [Fly.io](https://fly.io) with a persistent volume for SQLite.
+
+```bash
+cd packages/indexer
+
+# Create the app (first time only)
+fly apps create situation-indexer
+
+# Create a persistent volume for the database
+fly volumes create indexer_data --region iad --size 1
+
+# Set secrets
+fly secrets set VOYAGER_API_KEY=your-key ADMIN_API_KEY=your-key
+
+# Deploy (uses Dockerfile.indexer at repo root)
+fly deploy
+
+# Verify
+curl https://situation-indexer.fly.dev/health
+```
+
+The `fly.toml` is pre-configured with:
+- `shared-cpu-1x` / 512MB RAM
+- Persistent volume mounted at `/data`
+- `auto_stop_machines = false` (always-on for polling)
+- HTTPS enforced
+
+### Useful Fly commands
+
+```bash
+# View logs
+fly logs
+
+# SSH into the machine
+fly ssh console
+
+# Restart the app
+fly machines restart
+
+# Check volume
+fly volumes list
+```

package/dist/api/app.d.ts

@@ -0,0 +1,380 @@
+/**
+ * Elysia app factory — assembles all middleware and routes.
+ */
+import { Elysia } from 'elysia';
+import type { DatabaseService } from '../services/Database';
+import type { EventBusService } from '../services/EventBus';
+export interface AppConfig {
+    readonly db: DatabaseService;
+    readonly eventBus: EventBusService;
+    readonly adminApiKey: string;
+}
+export declare function createApp(config: AppConfig): Elysia<"", {
+    decorator: {};
+    store: {};
+    derive: {};
+    resolve: {};
+}, {
+    typebox: {};
+    error: {};
+} & {
+    typebox: {};
+    error: {};
+}, {
+    schema: {};
+    standaloneSchema: {};
+    macro: {};
+    macroFn: {};
+    parser: {};
+    response: {};
+} & {
+    schema: {};
+    standaloneSchema: {};
+    macro: {};
+    macroFn: {};
+    parser: {};
+    response: {};
+}, {
+    health: {
+        get: {
+            body: unknown;
+            params: {};
+            query: unknown;
+            headers: unknown;
+            response: {
+                200: import("../types").HealthResponse;
+            };
+        };
+    };
+} & {
+    api: {
+        markets: {
+            get: {
+                body: unknown;
+                params: {};
+                query: unknown;
+                headers: unknown;
+                response: {
+                    200: import("../types").MarketResponse[];
+                };
+            };
+        };
+    };
+} & {
+    api: {
+        markets: {
+            ":address": {
+                get: {
+                    body: unknown;
+                    params: {
+                        address: string;
+                    } & {};
+                    query: unknown;
+                    headers: unknown;
+                    response: {
+                        200: import("../types").MarketResponse | {
+                            error: string;
+                        };
+                        422: {
+                            type: "validation";
+                            on: string;
+                            summary?: string;
+                            message?: string;
+                            found?: unknown;
+                            property?: string;
+                            expected?: string;
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    api: {
+        markets: {
+            ":address": {
+                events: {
+                    get: {
+                        body: unknown;
+                        params: {
+                            address: string;
+                        } & {};
+                        query: unknown;
+                        headers: unknown;
+                        response: {
+                            200: import("../types").PaginatedResponse<import("../types").EventResponse>;
+                            422: {
+                                type: "validation";
+                                on: string;
+                                summary?: string;
+                                message?: string;
+                                found?: unknown;
+                                property?: string;
+                                expected?: string;
+                            };
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    api: {
+        markets: {
+            ":address": {
+                traders: {
+                    get: {
+                        body: unknown;
+                        params: {
+                            address: string;
+                        } & {};
+                        query: unknown;
+                        headers: unknown;
+                        response: {
+                            200: import("../types").TraderPositionResponse[];
+                            422: {
+                                type: "validation";
+                                on: string;
+                                summary?: string;
+                                message?: string;
+                                found?: unknown;
+                                property?: string;
+                                expected?: string;
+                            };
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    api: {
+        positions: {
+            ":trader": {
+                get: {
+                    body: unknown;
+                    params: {
+                        trader: string;
+                    } & {};
+                    query: unknown;
+                    headers: unknown;
+                    response: {
+                        200: import("../types").TraderPositionResponse[];
+                        422: {
+                            type: "validation";
+                            on: string;
+                            summary?: string;
+                            message?: string;
+                            found?: unknown;
+                            property?: string;
+                            expected?: string;
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    api: {
+        positions: {
+            ":trader": {
+                events: {
+                    get: {
+                        body: unknown;
+                        params: {
+                            trader: string;
+                        } & {};
+                        query: unknown;
+                        headers: unknown;
+                        response: {
+                            200: import("../types").PaginatedResponse<import("../types").TraderEventResponse>;
+                            422: {
+                                type: "validation";
+                                on: string;
+                                summary?: string;
+                                message?: string;
+                                found?: unknown;
+                                property?: string;
+                                expected?: string;
+                            };
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    api: {
+        positions: {
+            ":trader": {
+                stats: {
+                    get: {
+                        body: unknown;
+                        params: {
+                            trader: string;
+                        } & {};
+                        query: unknown;
+                        headers: unknown;
+                        response: {
+                            200: import("../types").TraderStatsResponse;
+                            422: {
+                                type: "validation";
+                                on: string;
+                                summary?: string;
+                                message?: string;
+                                found?: unknown;
+                                property?: string;
+                                expected?: string;
+                            };
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    api: {
+        rankings: {
+            get: {
+                body: unknown;
+                params: {};
+                query: unknown;
+                headers: unknown;
+                response: {
+                    200: import("../types").RankingEntry[];
+                };
+            };
+        };
+    };
+} & {
+    admin: {
+        markets: {
+            post: {
+                body: unknown;
+                params: {};
+                query: unknown;
+                headers: unknown;
+                response: {
+                    200: {
+                        error: string;
+                    } | {
+                        error: string;
+                        success?: never;
+                        market?: never;
+                    } | {
+                        success: boolean;
+                        market: import("../types").MarketRow;
+                        error?: never;
+                    };
+                };
+            };
+        };
+    };
+} & {
+    admin: {
+        markets: {
+            ":address": {
+                put: {
+                    body: unknown;
+                    params: {
+                        address: string;
+                    } & {};
+                    query: unknown;
+                    headers: unknown;
+                    response: {
+                        200: {
+                            error: string;
+                        } | {
+                            error: string;
+                            success?: never;
+                            market?: never;
+                        } | {
+                            success: boolean;
+                            market: import("../types").MarketRow;
+                            error?: never;
+                        };
+                        422: {
+                            type: "validation";
+                            on: string;
+                            summary?: string;
+                            message?: string;
+                            found?: unknown;
+                            property?: string;
+                            expected?: string;
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    admin: {
+        markets: {
+            ":address": {
+                delete: {
+                    body: unknown;
+                    params: {
+                        address: string;
+                    } & {};
+                    query: unknown;
+                    headers: unknown;
+                    response: {
+                        200: {
+                            error: string;
+                        } | {
+                            error: string;
+                            success?: never;
+                        } | {
+                            success: boolean;
+                            error?: never;
+                        };
+                        422: {
+                            type: "validation";
+                            on: string;
+                            summary?: string;
+                            message?: string;
+                            found?: unknown;
+                            property?: string;
+                            expected?: string;
+                        };
+                    };
+                };
+            };
+        };
+    };
+} & {
+    ws: {
+        subscribe: {
+            body: unknown;
+            params: {};
+            query: unknown;
+            headers: unknown;
+            response: {};
+        };
+    };
+}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+} & {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+} & {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+}>;
+//# sourceMappingURL=app.d.ts.map

package/dist/api/app.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../../src/api/app.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAe5D,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAgB,SAAS,CAAC,MAAM,EAAE,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAc1C"}

package/dist/api/app.js

@@ -0,0 +1,23 @@
+/**
+ * Elysia app factory — assembles all middleware and routes.
+ */
+import { Elysia } from 'elysia';
+import { cors } from '@elysiajs/cors';
+import { errorHandler } from './middleware/error-handler';
+import { healthRoutes, marketRoutes, marketEventsRoutes, marketTradersRoutes, positionRoutes, traderEventsRoutes, traderStatsRoutes, rankingRoutes, adminRoutes, wsRoutes, } from './routes';
+export function createApp(config) {
+    return new Elysia()
+        .use(cors())
+        .use(errorHandler)
+        .use(healthRoutes(config.db))
+        .use(marketRoutes(config.db))
+        .use(marketEventsRoutes(config.db))
+        .use(marketTradersRoutes(config.db))
+        .use(positionRoutes(config.db))
+        .use(traderEventsRoutes(config.db))
+        .use(traderStatsRoutes(config.db))
+        .use(rankingRoutes(config.db))
+        .use(adminRoutes(config.db, config.adminApiKey))
+        .use(wsRoutes(config.eventBus));
+}
+//# sourceMappingURL=app.js.map

package/dist/api/app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"app.js","sourceRoot":"","sources":["../../src/api/app.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAGtC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,YAAY,EACZ,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,QAAQ,GACT,MAAM,UAAU,CAAC;AAQlB,MAAM,UAAU,SAAS,CAAC,MAAiB;IACzC,OAAO,IAAI,MAAM,EAAE;SAChB,GAAG,CAAC,IAAI,EAAE,CAAC;SACX,GAAG,CAAC,YAAY,CAAC;SACjB,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SAC5B,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SAC5B,GAAG,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SAClC,GAAG,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SACnC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SAC9B,GAAG,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SAClC,GAAG,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SACjC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SAC7B,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;SAC/C,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/dist/api/middleware/admin-auth.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Bearer token check for admin endpoints.
+ */
+import { Elysia } from 'elysia';
+export declare const adminAuth: (adminApiKey: string) => Elysia<"", {
+    decorator: {};
+    store: {};
+    derive: {};
+    resolve: {};
+}, {
+    typebox: {};
+    error: {};
+}, {
+    schema: {};
+    standaloneSchema: {};
+    macro: {};
+    macroFn: {};
+    parser: {};
+    response: {};
+}, {}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {
+        200: {
+            error: string;
+        };
+    };
+}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+}>;
+//# sourceMappingURL=admin-auth.d.ts.map

package/dist/api/middleware/admin-auth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"admin-auth.d.ts","sourceRoot":"","sources":["../../../src/api/middleware/admin-auth.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,eAAO,MAAM,SAAS,GAAI,aAAa,MAAM;;;;;;;;;;;;;;;;;;;;;;mBAGZ,MAAM;;;;;;;;;EAapC,CAAC"}

package/dist/api/middleware/admin-auth.js

@@ -0,0 +1,18 @@
+/**
+ * Bearer token check for admin endpoints.
+ */
+import { Elysia } from 'elysia';
+export const adminAuth = (adminApiKey) => new Elysia({ name: 'admin-auth' }).onBeforeHandle({ as: 'scoped' }, ({ headers, set }) => {
+    const auth = headers['authorization'];
+    if (!auth || !auth.startsWith('Bearer ')) {
+        set.status = 401;
+        return { error: 'Missing or invalid Authorization header' };
+    }
+    const token = auth.slice(7);
+    if (token !== adminApiKey) {
+        set.status = 403;
+        return { error: 'Invalid API key' };
+    }
+    return undefined;
+});
+//# sourceMappingURL=admin-auth.js.map