@milaboratories/pframes-rs-serv 1.0.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md ADDED
@@ -0,0 +1,142 @@
1
+ # @milaboratories/pframes-serv
2
+
3
+ A high-performance HTTP server for serving Parquet files with full RFC 9110 compliance, authentication support, and optimized caching.
4
+
5
+ ## Overview
6
+
7
+ This package provides HTTP server functionality for the PFrames ecosystem, specifically designed for serving Parquet files with proper HTTP semantics. It implements a complete HTTP/1.1 server that handles range requests, conditional requests, authentication, and caching according to web standards.
8
+
9
+ ## Architecture
10
+
11
+ ### Core Components
12
+
13
+ - **Server** (`src/serve.ts`) - Generic HTTP server with lifecycle management
14
+ - **HTTP Handler** (`src/handler.ts`) - RFC 9110 compliant HTTP request handler
15
+ - **File System Store** (`src/fs-store.ts`) - Reader for local filesystem
16
+
17
+ ### Exports
18
+
19
+ #### Public API
20
+
21
+ - `serve()` - Main server function
22
+ - `createRequestHandler()` - HTTP request handler factory
23
+ - `FileSystemStore` - Local file system object store implementation
24
+
25
+ #### HttpHelpers (for re-export)
26
+
27
+ The `HttpHelpers` export contains utility functions intended for re-export through the `@milaboratories/pframes-node` package and consumption by PFrameDriver in public monorepo:
28
+
29
+ ### Binary Script
30
+
31
+ The `bin/parquet-server.mjs` script is used by the `core` package tests via the `ParquetServer` helper class for integration testing. It provides a standalone server that can be spawned programmatically for testing HTTP compliance and file serving functionality.
32
+
33
+ ## HTTP Handler Flow
34
+
35
+ The request handler implements a complete HTTP/1.1 server following [RFC 9110 (HTTP Semantics)](https://datatracker.ietf.org/doc/html/rfc9110) and [RFC 9111 (HTTP Caching)](https://datatracker.ietf.org/doc/html/rfc9111). Here's the detailed request processing flow:
36
+
37
+ ## Request Processing Flow
38
+
39
+ ```text
40
+ ┌─────────────────┐
41
+ │ Request Arrives │
42
+ └─────────┬───────┘
43
+
44
+
45
+ ┌─────────────────────┐
46
+ │ Set Default Headers │ ← Content-Length: 0, Date: now
47
+ │ (All Responses) │
48
+ └─────────┬───────────┘
49
+
50
+
51
+ ┌─────────────────────┐ [FAIL] No Token
52
+ │ Check Authentication│────────────┐
53
+ └─────────┬───────────┘ │
54
+ │ [PASS] Authorized │
55
+ ▼ ▼
56
+ ┌─────────────────────┐ ┌──────────────┐
57
+ │ Set Cache Headers │ │ Return 401 │ ← WWW-Authenticate: Bearer
58
+ └─────────┬───────────┘ │ Unauthorized │
59
+ │ └──────────────┘
60
+
61
+ ┌─────────────────────┐ [FAIL] Not GET/HEAD
62
+ │ Check HTTP Method │────────────┐
63
+ └─────────┬───────────┘ │
64
+ │ [PASS] Valid Method │
65
+ ▼ ▼
66
+ ┌─────────────────────┐ ┌──────────────┐
67
+ │ Parse URL │ │ Return 405 │ ← Allow: GET, HEAD
68
+ └─────────┬───────────┘ │ Not Allowed │
69
+ │ └──────────────┘
70
+
71
+ ┌─────────────────────┐ [FAIL] Invalid Pattern
72
+ │ Validate .parquet │────────────┐
73
+ │ Extension │ │
74
+ └─────────┬───────────┘ │
75
+ │ [PASS] Valid .parquet │
76
+ ▼ ▼
77
+ ┌─────────────────────┐ ┌──────────────┐
78
+ │ Set Content Headers │ │ Return 410 │ ← Permanently Gone
79
+ │ (Accept-Ranges, │ │ Gone │
80
+ │ Content-Type) │ └──────────────┘
81
+ └─────────┬───────────┘
82
+
83
+
84
+ ┌─────────────────────┐ [FAIL] File Missing
85
+ │ Check File Exists │────────────┐
86
+ └─────────┬───────────┘ │
87
+ │ [PASS] File Found │
88
+ ▼ ▼
89
+ ┌─────────────────────┐ ┌──────────────┐
90
+ │ Generate ETag │ │ Return 404 │ ← File Not Found
91
+ │ Set ETag Header │ │ Not Found │
92
+ └─────────┬───────────┘ └──────────────┘
93
+
94
+
95
+ ┌─────────────────────┐ [FAIL] Precondition Failed
96
+ │ Check If-Match │────────────┐
97
+ │ Conditional Headers │ │
98
+ └─────────┬───────────┘ │
99
+ │ [PASS] Conditions Met │
100
+ ▼ ▼
101
+ ┌─────────────────────┐ ┌──────────────┐
102
+ │ Check If-None-Match │ │ Return 412 │ ← Precondition Failed
103
+ └─────────┬───────────┘ │ Precondition │
104
+ │ │ Failed │
105
+ ▼ └──────────────┘
106
+ │ [FAIL] Not Modified
107
+ ├────────────────────────┐
108
+ │ ▼
109
+ │ ┌──────────────┐
110
+ │ │ Return 304 │ ← ETag, Not Modified
111
+ │ │ Not Modified │
112
+ │ └──────────────┘
113
+ │ [PASS] Modified
114
+
115
+ ┌─────────────────────┐
116
+ │ Parse Range Header │
117
+ └─────────┬───────────┘
118
+
119
+
120
+ ┌─────────────────────┐ [FAIL] Invalid Range
121
+ │ Validate Range │────────────┐
122
+ │ Against File Size │ │
123
+ └─────────┬───────────┘ │
124
+ │ [PASS] Valid Range │
125
+ ▼ ▼
126
+ ┌─────────────────────┐ ┌──────────────┐
127
+ │ Set Status & Length │ │ Return 416 │ ← Content-Range: bytes */size
128
+ │ 200 (full) or │ │ Range Not │
129
+ │ 206 (partial) │ │ Satisfiable │
130
+ └─────────┬───────────┘ └──────────────┘
131
+
132
+
133
+ ┌─────────────────────┐ [HEAD] Request
134
+ │ Check Request Type │────────────┐
135
+ └─────────┬───────────┘ │
136
+ │ [GET] Request │
137
+ ▼ ▼
138
+ ┌─────────────────────┐ ┌──────────────┐
139
+ │ Stream File Content │ │ Send Headers │ ← Headers Only
140
+ │ (with range support)│ │ Only (HEAD) │
141
+ └─────────────────────┘ └──────────────┘
142
+ ```
@@ -0,0 +1,3 @@
1
+ @echo off
2
+
3
+ node "%~dp0\parquet-server" %*
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ import { runParquetServer } from '../dist/index.js';
5
+ runParquetServer(); // all the code is in the TS file
package/package.json ADDED
@@ -0,0 +1,59 @@
1
+ {
2
+ "name": "@milaboratories/pframes-rs-serv",
3
+ "description": "PFrames - Node.js HTTP(S) Parquet Server",
4
+ "version": "1.0.61",
5
+ "type": "module",
6
+ "types": "./dist/index.d.ts",
7
+ "main": "./dist/index.js",
8
+ "exports": {
9
+ ".": {
10
+ "types": "./dist/index.d.ts",
11
+ "import": "./dist/index.js",
12
+ "require": "./dist/index.cjs"
13
+ }
14
+ },
15
+ "bin": {
16
+ "parquet-server": "./bin/parquet-server.mjs"
17
+ },
18
+ "files": [
19
+ "src/**/*",
20
+ "dist/**/*",
21
+ "bin/**/*"
22
+ ],
23
+ "devDependencies": {
24
+ "@milaboratories/helpers": "1.6.22",
25
+ "@milaboratories/pl-model-common": "1.19.13",
26
+ "@milaboratories/pl-model-middle-layer": "1.8.15",
27
+ "@milaboratories/ts-builder": "1.0.5",
28
+ "@milaboratories/ts-configs": "1.0.6",
29
+ "@types/autocannon": "^7.12.7",
30
+ "@types/node": "^20.19.11",
31
+ "@vitest/coverage-istanbul": "^3.2.4",
32
+ "autocannon": "^8.0.0",
33
+ "commander": "^14.0.0",
34
+ "selfsigned": "^3.0.1",
35
+ "typescript": "^5.9.2",
36
+ "undici": "^7.14.0",
37
+ "vite": "^7.1.3",
38
+ "vitest": "^3.2.4"
39
+ },
40
+ "license": "UNLICENSED",
41
+ "publishConfig": {
42
+ "registry": "https://npm.pkg.github.com"
43
+ },
44
+ "homepage": "https://github.com/milaboratory/pframes-rs#readme",
45
+ "repository": {
46
+ "type": "git",
47
+ "url": "git://github.com/milaboratory/pframes-rs.git"
48
+ },
49
+ "scripts": {
50
+ "ts-build": "pnpm exec ts-builder build --target node",
51
+ "build": "pnpm run ts-build",
52
+ "pretest": "pnpm run build",
53
+ "test": "pnpm exec vitest",
54
+ "dev-test": "pnpm run test",
55
+ "coverage-vscode": "pnpm exec vitest run --coverage",
56
+ "precoverage-html": "pnpm run coverage-vscode",
57
+ "coverage-html": "pnpm exec vite preview --outDir ./tests/coverage/lcov-report"
58
+ }
59
+ }
package/src/export.ts ADDED
@@ -0,0 +1,23 @@
1
+ import { RequestListener } from 'http';
2
+ import { FileSystemStore } from './fs-store';
3
+ import { createRequestHandler } from './handler';
4
+ import { serve } from './serve';
5
+ import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
6
+
7
+ export const HttpHelpers: PFrameInternal.HttpHelpers = {
8
+ createFsStore: async (
9
+ options: PFrameInternal.FsStoreOptions
10
+ ): Promise<PFrameInternal.ObjectStore> => {
11
+ return await FileSystemStore.init(options);
12
+ },
13
+ createRequestHandler: (
14
+ options: PFrameInternal.RequestHandlerOptions
15
+ ): RequestListener => {
16
+ return createRequestHandler(options);
17
+ },
18
+ createHttpServer: async (
19
+ options: PFrameInternal.HttpServerOptions
20
+ ): Promise<PFrameInternal.HttpServer> => {
21
+ return await serve(options);
22
+ }
23
+ };
@@ -0,0 +1,110 @@
1
+ import type { Readable } from 'node:stream';
2
+ import { createReadStream } from 'node:fs';
3
+ import { stat, open, type FileHandle } from 'node:fs/promises';
4
+ import { join, resolve } from 'node:path';
5
+ import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
6
+ import { ensureError } from '@milaboratories/pl-model-common';
7
+
8
+ /** Object store for serving files from a local directory */
9
+ export class FileSystemStore extends PFrameInternal.ObjectStore {
10
+ private readonly rootDir: string;
11
+
12
+ private constructor(options: PFrameInternal.FsStoreOptions) {
13
+ super(options);
14
+
15
+ this.rootDir = options.rootDir;
16
+ }
17
+
18
+ static async init(
19
+ options: PFrameInternal.FsStoreOptions
20
+ ): Promise<FileSystemStore> {
21
+ const resolvedRootDir = resolve(options.rootDir);
22
+
23
+ const rootStats = await stat(resolvedRootDir).catch(() => {
24
+ throw new Error(
25
+ `File system store root directory does not exist: ${resolvedRootDir}`
26
+ );
27
+ });
28
+ if (!rootStats.isDirectory()) {
29
+ throw new Error(
30
+ `File system store root path is not a directory: ${resolvedRootDir}`
31
+ );
32
+ }
33
+
34
+ return new FileSystemStore({
35
+ ...options,
36
+ rootDir: resolvedRootDir
37
+ });
38
+ }
39
+
40
+ override async request(
41
+ filename: PFrameInternal.ParquetFileName,
42
+ params: {
43
+ method: PFrameInternal.HttpMethod;
44
+ range?: PFrameInternal.HttpRange;
45
+ signal: AbortSignal;
46
+ callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;
47
+ }
48
+ ): Promise<void> {
49
+ let file: FileHandle;
50
+ try {
51
+ const path = join(this.rootDir, filename);
52
+ file = await open(path, 'r');
53
+ } catch (error: unknown) {
54
+ this.logger(
55
+ 'error',
56
+ `File system store failed to open file ${filename}: ${ensureError(error)}`
57
+ );
58
+ return await params.callback({ type: 'NotFound' });
59
+ }
60
+
61
+ try {
62
+ let size: number;
63
+ try {
64
+ ({ size } = await file.stat());
65
+ } catch (error: unknown) {
66
+ this.logger(
67
+ 'error',
68
+ `File system store failed to get size of file ${filename}: ${ensureError(error)}`
69
+ );
70
+ return await params.callback({ type: 'InternalError' });
71
+ }
72
+
73
+ const range = this.translate(size, params.range);
74
+ if (!range) {
75
+ return await params.callback({ type: 'RangeNotSatisfiable', size });
76
+ }
77
+
78
+ if (params.method === 'HEAD') {
79
+ return await params.callback({ type: 'Ok', size, range });
80
+ }
81
+
82
+ let data: Readable;
83
+ try {
84
+ data = createReadStream('ignored', {
85
+ fd: file.fd,
86
+ autoClose: false,
87
+ start: range.start,
88
+ end: range.end,
89
+ signal: params.signal
90
+ });
91
+ this.logger(
92
+ 'info',
93
+ `File system store created read stream for ${filename}[${range.start}..=${range.end}]`
94
+ );
95
+ } catch (error: unknown) {
96
+ this.logger(
97
+ 'error',
98
+ `File system store failed to create read stream for ${filename}[${range.start}..=${range.end}]: ${ensureError(error)}`
99
+ );
100
+ return await params.callback({ type: 'InternalError' });
101
+ }
102
+
103
+ return await params
104
+ .callback({ type: 'Ok', size, range, data })
105
+ .catch(() => {});
106
+ } finally {
107
+ await file.close();
108
+ }
109
+ }
110
+ }
package/src/handler.ts ADDED
@@ -0,0 +1,207 @@
1
+ import type {
2
+ IncomingMessage,
3
+ RequestListener,
4
+ ServerResponse
5
+ } from 'node:http';
6
+ import { pipeline } from 'node:stream/promises';
7
+ import { timingSafeEqual } from 'node:crypto';
8
+ import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
9
+ import {
10
+ createETag,
11
+ getFilenameFromUrl,
12
+ parseRange,
13
+ isGetOrHead,
14
+ isHead,
15
+ Options,
16
+ StatusCode,
17
+ HeaderName,
18
+ HeaderValue,
19
+ isGet
20
+ } from './utils';
21
+
22
+ /** Main request handler for parquet files */
23
+ async function handleRequest(
24
+ request: IncomingMessage,
25
+ response: ServerResponse,
26
+ store: PFrameInternal.ObjectStore
27
+ ): Promise<void> {
28
+ // RFC 9110 section 6.6.1: Date header should be present in all responses
29
+ response.sendDate = true;
30
+ // RFC 9110 section 8.6: Content-Length 0 as default for error responses
31
+ response.strictContentLength = true;
32
+ response.setHeader(HeaderName.ContentLength, 0);
33
+ // Note: setting Content-Length disables Node.js default Transfer-Encoding: chunked
34
+
35
+ // RFC 9111 section 5.2: Cache-Control header with public allows to cache authenticated responses
36
+ response.setHeader(HeaderName.CacheControl, HeaderValue.CacheControl);
37
+
38
+ // RFC 9110 section 15.5.6: Method not allowed
39
+ const method = request.method;
40
+ if (!isGetOrHead(method)) {
41
+ response.setHeader(HeaderName.Allow, HeaderValue.Allow);
42
+ return void response.writeHead(StatusCode.MethodNotAllowed).end();
43
+ }
44
+
45
+ const filename = getFilenameFromUrl(request);
46
+ if (filename === null) {
47
+ return void response.writeHead(StatusCode.Gone).end();
48
+ }
49
+
50
+ // From now on we are sure that the response would be a Parquet file
51
+ response.setHeader(HeaderName.AcceptRanges, HeaderValue.AcceptRanges);
52
+ response.setHeader(HeaderName.ContentType, HeaderValue.ContentType);
53
+
54
+ // RFC 9110 section 8.8.3: ETag header is used for cache versioning
55
+ const etag = createETag(filename);
56
+ // RFC 9110 section 8.8.2: Last-Modified header field for cache validation
57
+ const mtime = new Date(0); // Using fake fixed date since files are immutable
58
+ // RFC 9111 section 5.2: Cache-Control header with public allows to cache authenticated responses
59
+ response.setHeader(HeaderName.CacheControl, HeaderValue.CacheControl);
60
+ response.setHeader(HeaderName.ETag, etag);
61
+ response.setHeader(HeaderName.LastModified, mtime.toUTCString());
62
+
63
+ const options = new Options(request);
64
+ // RFC 9110 section 13.1.1: If-Match precondition evaluation
65
+ // RFC 9110 section 13.1.4: If-Unmodified-Since precondition evaluation
66
+ if (options.preconditionFailed(etag, mtime)) {
67
+ return void response.writeHead(StatusCode.PreconditionFailed).end();
68
+ }
69
+ // RFC 9110 section 13.1.2: If-None-Match precondition evaluation
70
+ // RFC 9110 section 13.1.3: If-Modified-Since precondition evaluation
71
+ else if (options.notModified(etag, mtime)) {
72
+ return void response.writeHead(StatusCode.NotModified).end();
73
+ }
74
+
75
+ const range = parseRange(request);
76
+ if (range === null) {
77
+ return void response.writeHead(StatusCode.BadRequest).end();
78
+ }
79
+
80
+ const abortController = new AbortController();
81
+ request.on('close', () => abortController.abort());
82
+ const signal = abortController.signal;
83
+
84
+ store.request(filename, {
85
+ method: 'GET',
86
+ range,
87
+ signal,
88
+ // pipeline automatically destroys the streams if they were not gracefully closed
89
+ callback: async (result) => {
90
+ if (response.destroyed) return void response.destroy();
91
+
92
+ switch (result.type) {
93
+ case 'InternalError':
94
+ // object store encountered network error, retry by client can help
95
+ return void response.writeHead(StatusCode.InternalServerError).end();
96
+ case 'NotFound':
97
+ // RFC 9110 section 15.4.5: Not found
98
+ return void response.writeHead(StatusCode.NotFound).end();
99
+ case 'RangeNotSatisfiable':
100
+ // RFC 9110 section 15.5.17: Range not satisfiable
101
+ response.setHeader(HeaderName.ContentRange, `bytes */${result.size}`);
102
+ return void response.writeHead(StatusCode.RangeNotSatisfiable).end();
103
+ case 'Ok':
104
+ break;
105
+ }
106
+
107
+ if (isGet(method) && !result.data) {
108
+ // object store implementation is incorrect, retry by client cannot help
109
+ return void response.writeHead(StatusCode.GatewayTimeout).end();
110
+ }
111
+
112
+ if (range) {
113
+ // RFC 9110 section 14.4: Partial content response
114
+ response.setHeader(
115
+ HeaderName.ContentLength,
116
+ result.range.end - result.range.start + 1
117
+ );
118
+ response.setHeader(
119
+ HeaderName.ContentRange,
120
+ `bytes ${result.range.start}-${result.range.end}/${result.size}`
121
+ );
122
+ response.writeHead(StatusCode.PartialContent);
123
+ } else {
124
+ // RFC 9110 section 15.3.1: OK response
125
+ response.setHeader(HeaderName.ContentLength, result.size);
126
+ response.writeHead(StatusCode.Ok);
127
+ }
128
+
129
+ // RFC 9110 section 9.3.2: HEAD method must not return message body
130
+ if (isHead(method)) {
131
+ return void response.end();
132
+ }
133
+
134
+ return await pipeline(result.data!, response, { signal }).catch(() => {
135
+ // Pipeline errors are expected when request is aborted or connection is lost
136
+ // Response head was already written, so we can't change status code
137
+ // Just mute the error - pipeline destroys the response stream
138
+ });
139
+ }
140
+ });
141
+ }
142
+
143
+ /**
144
+ * Create a request handler for serving files from an object store
145
+ * compatible with HTTP/1.1 as defined in RFC 9110 and RFC 9111:
146
+ * - <https://datatracker.ietf.org/doc/html/rfc9110>
147
+ * - <https://datatracker.ietf.org/doc/html/rfc9111>
148
+ *
149
+ * Accepts only paths of the form `/<filename>.parquet`, returns 404 otherwise
150
+ * Assumes that files are immutable (and sets cache headers accordingly)
151
+ */
152
+ export function createRequestHandler(
153
+ options: PFrameInternal.RequestHandlerOptions
154
+ ): RequestListener {
155
+ const { store } = options;
156
+ return (request, response) => void handleRequest(request, response, store);
157
+ }
158
+
159
+ /** Request authorization middleware */
160
+ function authorizeRequest(
161
+ request: IncomingMessage,
162
+ response: ServerResponse,
163
+ handler: RequestListener,
164
+ authHeader: string
165
+ ): void {
166
+ // RFC 9110 section 6.6.1: Date header should be present in all responses
167
+ response.sendDate = true;
168
+ // RFC 9110 section 8.6: Content-Length 0 as default for error responses
169
+ response.strictContentLength = true;
170
+ response.setHeader(HeaderName.ContentLength, 0);
171
+ // Note: setting Content-Length disables Node.js default Transfer-Encoding: chunked
172
+
173
+ const actualHeader = request.headers['authorization'];
174
+
175
+ // Early length check to avoid unnecessary processing
176
+ if (!actualHeader || actualHeader.length !== authHeader.length) {
177
+ // RFC 9110 section 11.6.1: WWW-Authenticate header field
178
+ response.setHeader(HeaderName.WWWAuthenticate, HeaderValue.WWWAuthenticate);
179
+ return void response.writeHead(StatusCode.Unauthorized).end();
180
+ }
181
+
182
+ // Use timing-safe comparison to prevent timing attacks
183
+ // <https://developers.cloudflare.com/workers/examples/protect-against-timing-attacks/>
184
+ const encoder = new TextEncoder();
185
+ const receivedBuffer = encoder.encode(actualHeader);
186
+ const expectedBuffer = encoder.encode(authHeader);
187
+
188
+ if (
189
+ receivedBuffer.byteLength !== expectedBuffer.byteLength ||
190
+ !timingSafeEqual(receivedBuffer, expectedBuffer)
191
+ ) {
192
+ response.setHeader(HeaderName.WWWAuthenticate, HeaderValue.WWWAuthenticate);
193
+ return void response.writeHead(StatusCode.Unauthorized).end();
194
+ }
195
+
196
+ return handler(request, response);
197
+ }
198
+
199
+ /** Apply Bearer token authorization to @param handler */
200
+ export function authorizeRequestHandler(
201
+ handler: RequestListener,
202
+ authToken: PFrameInternal.HttpAuthorizationToken
203
+ ): RequestListener {
204
+ const authHeader = `Bearer ${authToken}`;
205
+ return (request, response) =>
206
+ authorizeRequest(request, response, handler, authHeader);
207
+ }
package/src/index.ts ADDED
@@ -0,0 +1,5 @@
1
+ export * from './handler';
2
+ export * from './fs-store';
3
+ export * from './serve';
4
+ export * from './export';
5
+ export * from './parquet-server';
@@ -0,0 +1,58 @@
1
+ import { Command } from 'commander';
2
+ import { FileSystemStore } from './fs-store';
3
+ import { createRequestHandler } from './handler';
4
+ import { serve } from './serve';
5
+
6
+ /**
7
+ * Serves parquet files from the given root directory.
8
+ * Manages the server lifecycle with graceful shutdown.
9
+ */
10
+ export async function runParquetServer(): Promise<void> {
11
+ const program = new Command();
12
+
13
+ program
14
+ .name('parquet-server')
15
+ .description('Serve parquet files from a directory over HTTP(S)')
16
+ .argument('<root-directory>', 'Root directory containing parquet files')
17
+ .option('--http', 'Use HTTP instead of HTTPS', false)
18
+ .option('--no-auth', 'Disable authentication')
19
+ .action(
20
+ async (rootDir: string, options: { http: boolean; auth: boolean }) => {
21
+ const abortController = new AbortController();
22
+ process
23
+ .on('SIGINT', () => abortController.abort())
24
+ .on('SIGTERM', () => abortController.abort());
25
+ abortController.signal.throwIfAborted();
26
+
27
+ const store = await FileSystemStore.init({
28
+ rootDir,
29
+ logger: (level, message) => {
30
+ const timestamp = new Date(Date.now()).toISOString();
31
+ console.log(`[${timestamp}] [${level}] ${message}`);
32
+ }
33
+ });
34
+ abortController.signal.throwIfAborted();
35
+ const handler = createRequestHandler({ store });
36
+
37
+ const server = await serve({
38
+ handler,
39
+ ...(options.http && { http: true }),
40
+ ...(!options.auth && { noAuth: true })
41
+ });
42
+ abortController.signal.onabort = () => server.stop();
43
+ abortController.signal.throwIfAborted();
44
+
45
+ console.log(
46
+ JSON.stringify({
47
+ url: server.address,
48
+ ...(server.authToken && { authToken: server.authToken }),
49
+ ...(server.encodedCaCert && { caCert: server.encodedCaCert })
50
+ })
51
+ );
52
+
53
+ await server.stopped;
54
+ }
55
+ );
56
+
57
+ await program.parseAsync();
58
+ }
package/src/serve.ts ADDED
@@ -0,0 +1,159 @@
1
+ import {
2
+ createServer as createHttpServer,
3
+ RequestListener,
4
+ type Server as HttpServer,
5
+ type ServerOptions
6
+ } from 'node:http';
7
+ import {
8
+ createServer as createHttpsServer,
9
+ Server as HttpsServer
10
+ } from 'node:https';
11
+ import { AddressInfo } from 'node:net';
12
+ import { Deferred } from '@milaboratories/helpers';
13
+ import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
14
+ import {
15
+ base64Encode,
16
+ Base64Encoded,
17
+ ensureError,
18
+ PFrameError
19
+ } from '@milaboratories/pl-model-common';
20
+ import { generate, type GenerateResult } from 'selfsigned';
21
+ import { randomUUID } from 'node:crypto';
22
+ import { authorizeRequestHandler } from './handler';
23
+
24
+ /** Generate a self-signed certificate for localhost */
25
+ async function generateCertificate(): Promise<GenerateResult> {
26
+ const generateResult = new Deferred<GenerateResult>();
27
+ generate(
28
+ [{ name: 'commonName', value: 'localhost' }],
29
+ {
30
+ keySize: 2048,
31
+ algorithm: 'sha256',
32
+ extensions: [
33
+ {
34
+ name: 'subjectAltName',
35
+ altNames: [
36
+ { type: 2, value: 'localhost' }, // DNS
37
+ { type: 7, ip: '127.0.0.1' }, // IPv4
38
+ { type: 7, ip: '::1' } // IPv6
39
+ ]
40
+ }
41
+ ]
42
+ },
43
+ (error, result) => {
44
+ if (error) {
45
+ generateResult.reject(error);
46
+ } else {
47
+ generateResult.resolve(result);
48
+ }
49
+ }
50
+ );
51
+ return await generateResult.promise;
52
+ }
53
+
54
+ /** Create an object store URL from the server address info. */
55
+ function createObjectStoreUrl(
56
+ info: AddressInfo,
57
+ http?: true
58
+ ): PFrameInternal.ObjectStoreUrl {
59
+ const protocol = http ? 'http' : 'https';
60
+ switch (info.family) {
61
+ case 'IPv4':
62
+ return `${protocol}://${info.address}:${info.port}/` as PFrameInternal.ObjectStoreUrl;
63
+ case 'IPv6':
64
+ return `${protocol}://[${info.address}]:${info.port}/` as PFrameInternal.ObjectStoreUrl;
65
+ default:
66
+ throw new PFrameError(
67
+ `PFrame helper HTTP(S) server bound to 'localhost' has unknown address family: ${info.family}`
68
+ );
69
+ }
70
+ }
71
+
72
+ /**
73
+ * Serve HTTP requests using the provided handler.
74
+ * Returns a promise that resolves when the server is stopped.
75
+ */
76
+ export async function serve({
77
+ handler,
78
+ port = 0,
79
+ http,
80
+ noAuth
81
+ }: PFrameInternal.HttpServerOptions): Promise<PFrameInternal.HttpServer> {
82
+ const started = new Deferred<PFrameInternal.HttpServer>();
83
+ try {
84
+ let stopped: Deferred<void> | null = null;
85
+
86
+ let authToken: PFrameInternal.HttpAuthorizationToken | undefined;
87
+ let effectiveHandler: RequestListener = handler;
88
+ if (!noAuth) {
89
+ authToken = randomUUID() as PFrameInternal.HttpAuthorizationToken;
90
+ effectiveHandler = authorizeRequestHandler(effectiveHandler, authToken);
91
+ }
92
+
93
+ // Create HTTP server
94
+ let certificateBase64:
95
+ | Base64Encoded<PFrameInternal.PemCertificate>
96
+ | undefined;
97
+ const defaultOptions: ServerOptions = {
98
+ keepAlive: true
99
+ };
100
+ let server: HttpServer | HttpsServer;
101
+
102
+ if (http) {
103
+ server = createHttpServer(defaultOptions, effectiveHandler);
104
+ } else {
105
+ const { cert, private: key, public: ca } = await generateCertificate();
106
+ certificateBase64 = base64Encode(cert as PFrameInternal.PemCertificate);
107
+ server = createHttpsServer(
108
+ { ...defaultOptions, cert, key, ca },
109
+ effectiveHandler
110
+ );
111
+ }
112
+
113
+ const abortController = new AbortController();
114
+ server
115
+ .on('listening', () => {
116
+ // Cast is safe by specification <https://nodejs.org/api/net.html#serveraddress>
117
+ const address = createObjectStoreUrl(
118
+ server.address() as AddressInfo,
119
+ http
120
+ );
121
+ stopped = new Deferred<void>();
122
+
123
+ started.resolve({
124
+ get address(): PFrameInternal.ObjectStoreUrl {
125
+ return address;
126
+ },
127
+ get authToken(): PFrameInternal.HttpAuthorizationToken | undefined {
128
+ return authToken;
129
+ },
130
+ get encodedCaCert():
131
+ | Base64Encoded<PFrameInternal.PemCertificate>
132
+ | undefined {
133
+ return certificateBase64;
134
+ },
135
+ get stopped(): Promise<void> {
136
+ return stopped!.promise;
137
+ },
138
+ stop(): Promise<void> {
139
+ abortController.abort();
140
+ return stopped!.promise;
141
+ }
142
+ });
143
+ })
144
+ .on('error', (err) => {
145
+ started.reject(err);
146
+ stopped?.reject(err);
147
+ })
148
+ .on('close', () => stopped?.resolve())
149
+ .listen({
150
+ host: 'localhost',
151
+ port,
152
+ signal: abortController.signal
153
+ });
154
+ } catch (error: unknown) {
155
+ started.reject(ensureError(error));
156
+ }
157
+
158
+ return started.promise;
159
+ }
@@ -0,0 +1,20 @@
1
+ import { Branded } from '@milaboratories/pl-model-common';
2
+ import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
3
+
4
+ /**
5
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-8.8.3>
6
+ *
7
+ * Examples:
8
+ *
9
+ * ```text
10
+ * ETag: "xyzzy"
11
+ * ETag: W/"xyzzy"
12
+ * ```
13
+ */
14
+ export type Etag = Branded<string, 'Etag'>;
15
+
16
+ export function createETag(filename: PFrameInternal.ParquetFileName): Etag {
17
+ // For immutable files, use URL-safe base64 encoded filename as ETag
18
+ const filenameETag = Buffer.from(filename, 'utf8').toString('base64url');
19
+ return `"${filenameETag}"` as Etag;
20
+ }
@@ -0,0 +1,16 @@
1
+ import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
2
+ import { IncomingMessage } from 'node:http';
3
+
4
+ const PARQUET_FILENAME_REGEX = /^\/([\w\-.]+.parquet)$/;
5
+
6
+ export function getFilenameFromUrl(
7
+ request: IncomingMessage
8
+ ): PFrameInternal.ParquetFileName | null {
9
+ const url = request.url;
10
+ if (url === undefined) return null;
11
+
12
+ const match = url.match(PARQUET_FILENAME_REGEX);
13
+ if (!match) return null;
14
+
15
+ return match[1] as PFrameInternal.ParquetFileName;
16
+ }
@@ -0,0 +1,31 @@
1
+ 'use strict';
2
+
3
+ /** HTTP header names used in the parquet server handler */
4
+ export const HeaderName = {
5
+ Accept: 'accept',
6
+ AcceptRanges: 'accept-ranges',
7
+ Allow: 'allow',
8
+ Authorization: 'authorization',
9
+ CacheControl: 'cache-control',
10
+ ContentLength: 'content-length',
11
+ ContentRange: 'content-range',
12
+ ContentType: 'content-type',
13
+ Date: 'date',
14
+ ETag: 'etag',
15
+ IfMatch: 'if-match',
16
+ IfModifiedSince: 'if-modified-since',
17
+ IfNoneMatch: 'if-none-match',
18
+ IfUnmodifiedSince: 'if-unmodified-since',
19
+ LastModified: 'last-modified',
20
+ Range: 'range',
21
+ WWWAuthenticate: 'www-authenticate'
22
+ } as const;
23
+
24
+ /** HTTP header values used in the parquet server handler */
25
+ export const HeaderValue = {
26
+ AcceptRanges: 'bytes',
27
+ Allow: 'GET, HEAD',
28
+ CacheControl: 'public, immutable, max-age=31536000',
29
+ ContentType: 'application/octet-stream',
30
+ WWWAuthenticate: 'Bearer realm="parquet-server"'
31
+ } as const;
@@ -0,0 +1,7 @@
1
+ export * from './filename';
2
+ export * from './etag';
3
+ export * from './headers';
4
+ export * from './options';
5
+ export * from './range';
6
+ export * from './method';
7
+ export * from './status';
@@ -0,0 +1,20 @@
1
+ import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
2
+ import { IncomingHttpHeaders } from 'node:http';
3
+
4
+ export function isGetOrHead(
5
+ method: IncomingHttpHeaders['method']
6
+ ): method is PFrameInternal.HttpMethod {
7
+ return method === 'GET' || method === 'HEAD';
8
+ }
9
+
10
+ export function isGet(
11
+ method: PFrameInternal.HttpMethod
12
+ ): method is Extract<PFrameInternal.HttpMethod, 'GET'> {
13
+ return method === 'GET';
14
+ }
15
+
16
+ export function isHead(
17
+ method: PFrameInternal.HttpMethod
18
+ ): method is Extract<PFrameInternal.HttpMethod, 'HEAD'> {
19
+ return method === 'HEAD';
20
+ }
@@ -0,0 +1,152 @@
1
+ import { IncomingHttpHeaders, IncomingMessage } from 'node:http';
2
+ import { type Etag } from './etag';
3
+
4
+ type EtagMatchType =
5
+ | {
6
+ type: 'match';
7
+ value: Etag[];
8
+ }
9
+ | {
10
+ type: 'wildcard';
11
+ };
12
+
13
+ class EtagMatch {
14
+ private constructor(private readonly etagMatch: EtagMatchType) {}
15
+
16
+ static parse(etagMatch: string | undefined): EtagMatch | null {
17
+ if (etagMatch === undefined) return null;
18
+ if (etagMatch === '*') {
19
+ return new EtagMatch({ type: 'wildcard' });
20
+ } else {
21
+ return new EtagMatch({
22
+ type: 'match',
23
+ value: etagMatch.split(',').map((etag) => etag.trim() as Etag)
24
+ });
25
+ }
26
+ }
27
+
28
+ matches(etag: Etag): boolean {
29
+ switch (this.etagMatch.type) {
30
+ case 'match':
31
+ return this.etagMatch.value.includes(etag);
32
+ case 'wildcard':
33
+ return true;
34
+ }
35
+ }
36
+ }
37
+
38
+ class DateMatch {
39
+ private constructor(private readonly mtime: Date) {}
40
+
41
+ static parse(dateMatch: string | undefined): DateMatch | null {
42
+ if (dateMatch === undefined) return null;
43
+ const time = Date.parse(dateMatch);
44
+ if (isNaN(time)) return null;
45
+ return new DateMatch(new Date(time));
46
+ }
47
+
48
+ matches(mtime: Date): boolean {
49
+ return this.mtime.getTime() / 1_000 === mtime.getTime() / 1_000;
50
+ }
51
+ }
52
+
53
+ export class Options {
54
+ /**
55
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#name-if-match>
56
+ *
57
+ * Examples:
58
+ *
59
+ * ```text
60
+ * If-Match: "xyzzy"
61
+ * If-Match: "xyzzy", "r2d2xxxx", "c3piozzzz"
62
+ * If-Match: *
63
+ * ```
64
+ */
65
+ private ifMatch: EtagMatch | null;
66
+
67
+ /**
68
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-13.1.2>
69
+ *
70
+ * Examples:
71
+ *
72
+ * ```text
73
+ * If-None-Match: "xyzzy"
74
+ * If-None-Match: "xyzzy", "r2d2xxxx", "c3piozzzz"
75
+ * If-None-Match: *
76
+ * ```
77
+ */
78
+ private ifNoneMatch: EtagMatch | null;
79
+
80
+ /**
81
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-13.1.3>
82
+ *
83
+ * Examples:
84
+ *
85
+ * ```text
86
+ * If-Modified-Since: Mon, 07 Jan 2002 19:43:36 GMT
87
+ * ```
88
+ */
89
+ private ifModifiedSince: DateMatch | null;
90
+
91
+ /**
92
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-13.1.4>
93
+ *
94
+ * Examples:
95
+ *
96
+ * ```text
97
+ * If-Unmodified-Since: Mon, 07 Jan 2002 19:43:36 GMT
98
+ * ```
99
+ */
100
+ private ifUnmodifiedSince: DateMatch | null;
101
+
102
+ constructor(request: IncomingMessage) {
103
+ this.ifMatch = EtagMatch.parse(request.headers['if-match']);
104
+ this.ifNoneMatch = EtagMatch.parse(request.headers['if-none-match']);
105
+ this.ifModifiedSince = DateMatch.parse(
106
+ request.headers['if-modified-since']
107
+ );
108
+ this.ifUnmodifiedSince = DateMatch.parse(
109
+ request.headers['if-unmodified-since']
110
+ );
111
+ }
112
+
113
+ /**
114
+ * RFC 9110 section 13.1.1: If-Match precondition evaluation
115
+ * RFC 9110 section 13.1.4: If-Unmodified-Since precondition evaluation
116
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-13.1.1>
117
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-13.1.4>
118
+ */
119
+ preconditionFailed(etag: Etag, mtime: Date): boolean {
120
+ // If-Match precondition takes precedence
121
+ if (this.ifMatch !== null && !this.ifMatch.matches(etag)) {
122
+ return true;
123
+ }
124
+
125
+ // If-Unmodified-Since precondition (only if no If-Match header)
126
+ if (this.ifMatch === null && this.ifUnmodifiedSince !== null) {
127
+ return !this.ifUnmodifiedSince.matches(mtime);
128
+ }
129
+
130
+ return false;
131
+ }
132
+
133
+ /**
134
+ * RFC 9110 section 13.1.2: If-None-Match precondition evaluation
135
+ * RFC 9110 section 13.1.3: If-Modified-Since precondition evaluation
136
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-13.1.2>
137
+ * See <https://datatracker.ietf.org/doc/html/rfc9110#section-13.1.3>
138
+ */
139
+ notModified(etag: Etag, mtime: Date): boolean {
140
+ // If-None-Match takes precedence over If-Modified-Since
141
+ if (this.ifNoneMatch !== null) {
142
+ return this.ifNoneMatch.matches(etag);
143
+ }
144
+
145
+ // If-Modified-Since (only if no If-None-Match header)
146
+ if (this.ifModifiedSince !== null) {
147
+ return this.ifModifiedSince.matches(mtime);
148
+ }
149
+
150
+ return false;
151
+ }
152
+ }
@@ -0,0 +1,40 @@
1
+ import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
2
+ import { IncomingMessage } from 'node:http';
3
+
4
+ export function parseRange(
5
+ request: IncomingMessage
6
+ ): PFrameInternal.HttpRange | null | undefined {
7
+ const range = request.headers['range'];
8
+ if (range === undefined) return undefined;
9
+
10
+ const match = range.match(/^bytes=(\d*)-(\d*)$/);
11
+ if (!match) return null;
12
+
13
+ const [, startStr, endStr] = match;
14
+ const start = startStr ? parseInt(startStr, 10) : null;
15
+ const end = endStr ? parseInt(endStr, 10) : null;
16
+ if (
17
+ (start !== null && (isNaN(start) || start < 0)) ||
18
+ (end !== null && (isNaN(end) || end < 0)) ||
19
+ (start !== null && end !== null && start > end)
20
+ )
21
+ return null;
22
+
23
+ // Both start and end are specified - bounded range
24
+ if (start !== null && end !== null) {
25
+ return { type: 'bounded', start, end };
26
+ }
27
+
28
+ // Only start is specified - offset range (e.g., bytes=500-)
29
+ if (start !== null && end === null) {
30
+ return { type: 'offset', offset: start };
31
+ }
32
+
33
+ // Only end is specified - suffix range (e.g., bytes=-500)
34
+ if (start === null && end !== null) {
35
+ return { type: 'suffix', suffix: end };
36
+ }
37
+
38
+ // Neither start nor end specified (bytes=-) - invalid
39
+ return null;
40
+ }
@@ -0,0 +1,15 @@
1
+ /** HTTP status codes used in the parquet server handler */
2
+ export const StatusCode = {
3
+ Ok: 200, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/200>
4
+ PartialContent: 206, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/206>
5
+ NotModified: 304, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/304>
6
+ BadRequest: 400, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400>
7
+ Unauthorized: 401, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401>
8
+ NotFound: 404, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404>
9
+ MethodNotAllowed: 405, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/405>
10
+ Gone: 410, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/410>
11
+ PreconditionFailed: 412, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/412>
12
+ RangeNotSatisfiable: 416, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416>
13
+ InternalServerError: 500, // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/500>
14
+ GatewayTimeout: 504 // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/504>
15
+ } as const;