@drakkar.software/starfish-events 3.0.0-alpha.40

package/dist/plugin.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Starfish server plugin: intercepts JSON event-batch pushes and encodes them
+ * as Parquet files written directly to the object store (typically S3).
+ *
+ * ## How it works
+ *
+ * 1. Register a JSON-typed collection (allowedMimeTypes: ["application/json"])
+ *    with public write access.
+ * 2. Attach this plugin to the sync router.
+ * 3. Each push to that collection is intercepted here; the JSON event batch is
+ *    encoded as Parquet and stored via `store.putBytes`, short-circuiting the
+ *    default JSON document write so no JSON is persisted alongside the Parquet.
+ *
+ * ## Collection requirement
+ *
+ * The intercepted collection **must** be JSON-typed — `interceptPush` only
+ * receives a populated `rawBody` for JSON collections. A binary (parquet-typed)
+ * collection would yield an empty body.
+ *
+ * ## One file per batch
+ *
+ * Parquet's column-footer format makes in-place append impractical. Each
+ * `send()` call from the SunGlasses adapter writes a unique path (batchId in
+ * the storagePath template). DuckDB's `read_parquet('s3://…/**‌/*.parquet')`
+ * glob treats all files under the prefix as one logical dataset.
+ *
+ * ## Privacy
+ *
+ * Never log `distinct_id`, `properties`, or `context`. Log counts only.
+ * These values ride as opaque strings into Parquet.
+ */
+import type { ServerPlugin } from "@drakkar.software/starfish-protocol";
+import type { ObjectStore } from "@drakkar.software/starfish-server";
+/** Options for {@link createEventsServerPlugin}. */
+export interface EventsPluginOptions {
+    /**
+     * Object store the plugin writes Parquet files to.
+     * Must implement `putBytes` (e.g. `S3ObjectStore` from `starfish-server/s3`).
+     * Pass the **same** store instance that you pass to `createSyncRouter`.
+     */
+    store: ObjectStore;
+    /**
+     * Name of the collection to intercept.
+     * Must match the `name` field in the `SyncConfig.collections` entry.
+     * Example: `"events"`
+     */
+    collection: string;
+    /**
+     * Storage-path template for the output Parquet key.
+     * Supports `{param}` placeholders resolved from the push URL's path params.
+     * Example: `"events/{app}/{batchId}"` → `"events/myapp/<uuid>"`
+     *
+     * The plugin appends `.parquet` when the resolved key doesn't already end
+     * with it, so you can omit the extension from the template.
+     */
+    storagePath: string;
+}
+/**
+ * Create a Starfish server plugin that encodes SunGlasses event batches as
+ * Parquet and writes them to the object store.
+ *
+ * @example
+ * ```ts
+ * import { S3ObjectStore } from "@drakkar.software/starfish-server/s3"
+ * import { createEventsServerPlugin } from "@drakkar.software/starfish-events"
+ *
+ * const store = new S3ObjectStore({ bucket: "my-bucket", ... })
+ * const eventsPlugin = createEventsServerPlugin({
+ *   store,
+ *   collection: "events",
+ *   storagePath: "events/{app}/{batchId}",
+ * })
+ *
+ * const sync = createSyncRouter({
+ *   store,
+ *   config: {
+ *     version: 1,
+ *     collections: [{
+ *       name: "events",
+ *       storagePath: "events/{app}/{batchId}",
+ *       readRoles: ["public"],
+ *       writeRoles: ["public"],
+ *       encryption: "none",
+ *       allowedMimeTypes: ["application/json"],  // ← JSON-typed, not parquet
+ *       maxBodyBytes: 8_000_000,
+ *     }],
+ *   },
+ *   plugins: [eventsPlugin],
+ * })
+ * ```
+ */
+export declare function createEventsServerPlugin(opts: EventsPluginOptions): ServerPlugin;

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@drakkar.software/starfish-events",
+  "version": "3.0.0-alpha.40",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Drakkar-Software/starfish.git",
+    "directory": "packages/ts/events"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "Starfish server plugin that accepts JSON event batches and encodes them as Parquet on S3",
+  "type": "module",
+  "sideEffects": false,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "hyparquet-writer": "^0.16.1",
+    "@drakkar.software/starfish-protocol": "3.0.0-alpha.40",
+    "@drakkar.software/starfish-server": "3.0.0-alpha.40"
+  },
+  "devDependencies": {
+    "esbuild": "^0.27.4",
+    "hyparquet": "1.26.1",
+    "typescript": "^5.5.0",
+    "vitest": "^3.0.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "node build.mjs && tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  }
+}