@milaboratories/pl-model-middle-layer 1.8.13 → 1.8.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/dist/pframe/internal_api/common.d.ts +1 -0
  2. package/dist/pframe/internal_api/common.d.ts.map +1 -1
  3. package/dist/pframe/internal_api/http_helpers.d.ts +79 -52
  4. package/dist/pframe/internal_api/http_helpers.d.ts.map +1 -1
  5. package/dist/pframe/internal_api/pframe.d.ts +1 -1
  6. package/dist/pframe/internal_api/pframe.d.ts.map +1 -1
  7. package/package.json +3 -3
  8. package/src/pframe/internal_api/common.ts +6 -1
  9. package/src/pframe/internal_api/http_helpers.ts +105 -64
  10. package/src/pframe/internal_api/pframe.ts +1 -5
package/dist/pframe/internal_api/common.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { AxisSpec, ValueType } from '@milaboratories/pl-model-common';
2
+ export type Logger = (level: 'info' | 'warn' | 'error', message: string) => void;
2
3
  export interface SingleAxisSelector {
3
4
  name: string;
4
5
  type?: ValueType;
package/dist/pframe/internal_api/common.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,QAAQ,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAE9E,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,kBAAkB,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,cAAc,EAAE,iBAAiB,EAAE,CAAC;CACrC;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;CAC3B,CAAA"}
1
+ {"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG,CACnB,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,EAChC,OAAO,EAAE,MAAM,KACZ,IAAI,CAAC;AAEV,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,kBAAkB,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,cAAc,EAAE,iBAAiB,EAAE,CAAC;CACrC;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;CAC3B,CAAA"}
package/dist/pframe/internal_api/http_helpers.d.ts CHANGED
@@ -1,70 +1,97 @@
1
1
  import type { Readable } from 'node:stream';
2
2
  import type { RequestListener } from 'node:http';
3
3
  import type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';
4
- /** File range specification */
5
- export type FileRange = {
4
+ import type { Logger } from './common';
5
+ /** Parquet file name */
6
+ export type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;
7
+ /** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */
8
+ export type HttpRange = {
9
+ /**
10
+ * Get file content in the specified byte range
11
+ *
12
+ * @example
13
+ * ```
14
+ * GET /file.parquet HTTP/1.1
15
+ * Range: bytes=0-1023
16
+ * ```
17
+ */
18
+ type: 'bounded';
6
19
  /** Start byte position (inclusive) */
7
20
  start: number;
8
21
  /** End byte position (inclusive) */
9
22
  end: number;
10
- };
11
- /**
12
- * File system abstraction for request handler factory,
13
- * @see HttpHelpers.createRequestHandler.
14
- * Assumes that it is working with flat directory structure.
15
- * Accepts filenames with extension as input (e.g. `file.parquet`).
16
- */
17
- export interface ObjectStore {
23
+ } | {
18
24
  /**
19
- * @returns file size in bytes or `-1` if file does not exist or permissions do not allow access.
20
- * @throws if file can become accessible after retry (e.g. on network error)
25
+ * Get byte range starting from the specified offset
21
26
  *
22
27
  * @example
23
- * ```ts
24
- * async getFileSize(filename: string): Promise<number> {
25
- * const filePath = this.resolve(filename);
26
- * return await fs
27
- * .stat(filePath)
28
- * .then((stat) => ({ size: stat.isFile() ? stat.size : -1 }))
29
- * .catch(() => ({ size: -1 }));
30
- * }
28
+ * ```
29
+ * GET /file.parquet HTTP/1.1
30
+ * Range: bytes=1024-
31
31
  * ```
32
32
  */
33
- getFileSize(filename: string): Promise<number>;
33
+ type: 'offset';
34
+ /** Start byte position (inclusive) */
35
+ offset: number;
36
+ } | {
34
37
  /**
35
- * Execute action with readable stream (actions can be concurrency limited by the store).
36
- * Action resolves when stream is closed by handler @see HttpHelpers.createRequestHandler
37
- *
38
- * @param filename - existing file name (for which @see ObjectStore.getFileSize returned non-negative value)
39
- * @param range - valid range of bytes to read from the file (store may skip validation)
40
- * @param action - function to execute with the stream, responsible for closing the stream
41
- * @returns promise that resolves after the action is completed
38
+ * Get byte range starting from the specified suffix
42
39
  *
43
40
  * @example
44
- * ```ts
45
- * async withReadStream(params: {
46
- * filename: string;
47
- * range: FileRange;
48
- * action: (stream: Readable) => Promise<void>;
49
- * }): Promise<void> {
50
- * const { filename, range, action } = params;
51
- * const filePath = this.resolve(filename);
52
- *
53
- * try {
54
- * const stream = createReadStream(filePath, range);
55
- * return await action(stream);
56
- * } catch (err: unknown) {
57
- * console.error(`failed to create read stream for ${filename} - ${ensureError(err)}`);
58
- * throw;
59
- * }
60
- * }
61
41
  * ```
42
+ * GET /file.parquet HTTP/1.1
43
+ * Range: bytes=-1024
44
+ * ```
45
+ */
46
+ type: 'suffix';
47
+ /** End byte position (inclusive) */
48
+ suffix: number;
49
+ };
50
+ /** HTTP method passed to object store */
51
+ export type HttpMethod = 'GET' | 'HEAD';
52
+ /** HTTP response from object store */
53
+ export type ObjectStoreResponse<Method extends HttpMethod = HttpMethod> = {
54
+ /** Will be translated to 500 Internal Server Error by the handler */
55
+ type: 'InternalError';
56
+ } | {
57
+ /** Will be translated to 404 Not Found by the handler */
58
+ type: 'NotFound';
59
+ } | {
60
+ /** Will be translated to 416 Range Not Satisfiable by the handler */
61
+ type: 'RangeNotSatisfiable';
62
+ /** Total file size in bytes */
63
+ size: number;
64
+ } | {
65
+ /** Will be translated to 200 OK or 206 Partial Content by the handler */
66
+ type: 'Ok';
67
+ /** Total file size in bytes */
68
+ size: number;
69
+ /** Stream of file content, undefined for HEAD requests */
70
+ data: Method extends 'HEAD' ? undefined : Readable;
71
+ };
72
+ /** Common options for object store creation */
73
+ export interface ObjectStoreOptions {
74
+ /** Logger instance, no logging is performed when not provided */
75
+ logger?: Logger;
76
+ }
77
+ /** Options for file system object store creation */
78
+ export interface FsStoreOptions extends ObjectStoreOptions {
79
+ /** Local directory to serve files from */
80
+ rootDir: string;
81
+ }
82
+ /** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */
83
+ export interface ObjectStore {
84
+ /**
85
+ * Proxy HTTP(S) request for parquet file to object store.
86
+ * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler
87
+ * Callback API is used so that ObjectStore can limit the number of concurrent requests.
62
88
  */
63
- withReadStream(params: {
64
- filename: string;
65
- range: FileRange;
66
- action: (stream: Readable) => Promise<void>;
67
- }): Promise<void>;
89
+ request<Method extends HttpMethod>(filename: ParquetFileName, params: {
90
+ method: Method;
91
+ range?: HttpRange;
92
+ signal?: AbortSignal;
93
+ callback: (response: ObjectStoreResponse<Method>) => Promise<void>;
94
+ }): void;
68
95
  }
69
96
  /** Object store base URL in format accepted by Apache DataFusion and DuckDB */
70
97
  export type ObjectStoreUrl = Branded<string, 'PFrameInternal.ObjectStoreUrl'>;
@@ -125,7 +152,7 @@ export interface HttpHelpers {
125
152
  * Create an object store for serving files from a local directory.
126
153
  * Rejects if the provided path does not exist or is not a directory.
127
154
  */
128
- createFsStore(rootDir: string): Promise<ObjectStore>;
155
+ createFsStore(options: FsStoreOptions): Promise<ObjectStore>;
129
156
  /**
130
157
  * Create an HTTP request handler for serving files from an object store.
131
158
  * Accepts only paths of the form `/<filename>.parquet`, returns 410 otherwise.
@@ -140,7 +167,7 @@ export interface HttpHelpers {
140
167
  * ```ts
141
168
  * const rootDir = '/path/to/directory/with/parquet/files';
142
169
  *
143
- * let store = await HttpHelpers.createFsStore(rootDir).catch((err: unknown) => {
170
+ * let store = await HttpHelpers.createFsStore({ rootDir }).catch((err: unknown) => {
144
171
  * throw new Error(`Failed to create file store for ${rootDir} - ${ensureError(err)}`);
145
172
  * });
146
173
  *
package/dist/pframe/internal_api/http_helpers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"http_helpers.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAE9E,+BAA+B;AAC/B,MAAM,MAAM,SAAS,GAAG;IACtB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,cAAc,CAAC,MAAM,EAAE;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,SAAS,CAAC;QACjB,MAAM,EAAE,CAAC,MAAM,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;KAC7C,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnB;AAED,+EAA+E;AAC/E,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,+CAA+C;AAC/C,MAAM,MAAM,qBAAqB,GAAG;IAClC,uEAAuE;IACvE,KAAK,EAAE,WAAW,CAAC;CAEpB,CAAA;AAED,mCAAmC;AACnC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,8EAA8E;IAC9E,OAAO,EAAE,eAAe,CAAC;IACzB,uDAAuD;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,kFAAkF;IAClF,IAAI,CAAC,EAAE,IAAI,CAAC;CACb,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,EAAE,uCAAuC,CAAC,CAAC;AAE9F;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,kEAAkE;IAClE,IAAI,OAAO,IAAI,cAAc,CAAC;IAC9B,sGAAsG;IACtG,IAAI,SAAS,IAAI,sBAAsB,GAAG,SAAS,CAAC;IACpD,0GAA0G;IAC1G,IAAI,aAAa,IAAI,aAAa,CAAC,cAAc,CAAC,GAAG,SAAS,CAAC;IAC/D,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,+EAA+E;IAC/E,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAErD;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,eAAe,CAAC;IAEtE;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACnE"}
1
+ {"version":3,"file":"http_helpers.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAC9E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,wBAAwB;AACxB,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE,gCAAgC,CAAC,CAAC;AAE7F,2FAA2F;AAC3F,MAAM,MAAM,SAAS,GACjB;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,SAAS,CAAC;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,yCAAyC;AACzC,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,CAAC;AAExC,sCAAsC;AACtC,MAAM,MAAM,mBAAmB,CAAC,MAAM,SAAS,UAAU,GAAG,UAAU,IAClE;IACE,qEAAqE;IACrE,IAAI,EAAE,eAAe,CAAC;CACvB,GACD;IACE,yDAAyD;IACzD,IAAI,EAAE,UAAU,CAAC;CAClB,GACD;IACE,qEAAqE;IACrE,IAAI,EAAE,qBAAqB,CAAC;IAC5B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,yEAAyE;IACzE,IAAI,EAAE,IAAI,CAAC;IACX,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,IAAI,EAAE,MAAM,SAAS,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;CACpD,CAAA;AAEL,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,oDAAoD;AACpD,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iGAAiG;AACjG,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,OAAO,CAAC,MAAM,SAAS,UAAU,EAC/B,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,SAAS,CAAC;QAClB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,QAAQ,EAAE,CAAC,QAAQ,EAAE,mBAAmB,CAAC,MAAM,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;KACpE,GACA,IAAI,CAAC;CACT;AAED,+EAA+E;AAC/E,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,+CAA+C;AAC/C,MAAM,MAAM,qBAAqB,GAAG;IAClC,uEAAuE;IACvE,KAAK,EAAE,WAAW,CAAC;CAEpB,CAAA;AAED,mCAAmC;AACnC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,8EAA8E;IAC9E,OAAO,EAAE,eAAe,CAAC;IACzB,uDAAuD;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,kFAAkF;IAClF,IAAI,CAAC,EAAE,IAAI,CAAC;CACb,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,EAAE,uCAAuC,CAAC,CAAC;AAE9F;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,kEAAkE;IAClE,IAAI,OAAO,IAAI,cAAc,CAAC;IAC9B,sGAAsG;IACtG,IAAI,SAAS,IAAI,sBAAsB,GAAG,SAAS,CAAC;IACpD,0GAA0G;IAC1G,IAAI,aAAa,IAAI,aAAa,CAAC,cAAc,CAAC,GAAG,SAAS,CAAC;IAC/D,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,+EAA+E;IAC/E,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE7D;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,eAAe,CAAC;IAEtE;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACnE"}
package/dist/pframe/internal_api/pframe.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import type { PFrameFactoryAPIV2, PFrameFactoryAPIV3 } from './api_factory';
2
2
  import type { PFrameReadAPIV8, PFrameReadAPIV9 } from './api_read';
3
- export type Logger = (level: 'info' | 'warn' | 'error', message: string) => void;
3
+ import type { Logger } from './common';
4
4
  export interface PFrameV9 extends PFrameFactoryAPIV2, PFrameReadAPIV8 {
5
5
  }
6
6
  export interface PFrameV10 extends PFrameFactoryAPIV3, PFrameReadAPIV9 {
package/dist/pframe/internal_api/pframe.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"pframe.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/pframe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,MAAM,MAAM,MAAM,GAAG,CACnB,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,EAChC,OAAO,EAAE,MAAM,KACZ,IAAI,CAAC;AAEV,MAAM,WAAW,QAAS,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAExE,MAAM,WAAW,SAAU,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAEzE,MAAM,MAAM,aAAa,GAAG;IAC1B,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAA;AAED,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,CAAC;IAEhD;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;CACtC"}
1
+ {"version":3,"file":"pframe.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/pframe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,MAAM,WAAW,QAAS,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAExE,MAAM,WAAW,SAAU,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAEzE,MAAM,MAAM,aAAa,GAAG;IAC1B,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAA;AAED,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,CAAC;IAEhD;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;CACtC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@milaboratories/pl-model-middle-layer",
3
- "version": "1.8.13",
3
+ "version": "1.8.14",
4
4
  "description": "Common model between middle layer and non-block UI code",
5
5
  "type": "module",
6
6
  "types": "./dist/index.d.ts",
@@ -24,9 +24,9 @@
24
24
  },
25
25
  "devDependencies": {
26
26
  "typescript": "~5.6.3",
27
- "@milaboratories/build-configs": "1.0.8",
28
27
  "@milaboratories/ts-configs": "1.0.6",
29
- "@milaboratories/ts-builder": "1.0.5"
28
+ "@milaboratories/ts-builder": "1.0.5",
29
+ "@milaboratories/build-configs": "1.0.8"
30
30
  },
31
31
  "scripts": {
32
32
  "type-check": "ts-builder types --target node",
package/src/pframe/internal_api/common.ts CHANGED
@@ -1,4 +1,9 @@
1
- import { AxisId, AxisSpec, ValueType } from '@milaboratories/pl-model-common';
1
+ import { AxisSpec, ValueType } from '@milaboratories/pl-model-common';
2
+
3
+ export type Logger = (
4
+ level: 'info' | 'warn' | 'error',
5
+ message: string
6
+ ) => void;
2
7
 
3
8
  export interface SingleAxisSelector {
4
9
  name: string;
package/src/pframe/internal_api/http_helpers.ts CHANGED
@@ -1,73 +1,114 @@
1
1
  import type { Readable } from 'node:stream';
2
2
  import type { RequestListener } from 'node:http';
3
3
  import type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';
4
+ import type { Logger } from './common';
4
5
 
5
- /** File range specification */
6
- export type FileRange = {
7
- /** Start byte position (inclusive) */
8
- start: number;
9
- /** End byte position (inclusive) */
10
- end: number;
11
- };
6
+ /** Parquet file name */
7
+ export type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;
12
8
 
13
- /**
14
- * File system abstraction for request handler factory,
15
- * @see HttpHelpers.createRequestHandler.
16
- * Assumes that it is working with flat directory structure.
17
- * Accepts filenames with extension as input (e.g. `file.parquet`).
18
- */
19
- export interface ObjectStore {
20
- /**
21
- * @returns file size in bytes or `-1` if file does not exist or permissions do not allow access.
22
- * @throws if file can become accessible after retry (e.g. on network error)
23
- *
24
- * @example
25
- * ```ts
26
- * async getFileSize(filename: string): Promise<number> {
27
- * const filePath = this.resolve(filename);
28
- * return await fs
29
- * .stat(filePath)
30
- * .then((stat) => ({ size: stat.isFile() ? stat.size : -1 }))
31
- * .catch(() => ({ size: -1 }));
32
- * }
33
- * ```
34
- */
35
- getFileSize(filename: string): Promise<number>;
9
+ /** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */
10
+ export type HttpRange =
11
+ | {
12
+ /**
13
+ * Get file content in the specified byte range
14
+ *
15
+ * @example
16
+ * ```
17
+ * GET /file.parquet HTTP/1.1
18
+ * Range: bytes=0-1023
19
+ * ```
20
+ */
21
+ type: 'bounded';
22
+ /** Start byte position (inclusive) */
23
+ start: number;
24
+ /** End byte position (inclusive) */
25
+ end: number;
26
+ }
27
+ | {
28
+ /**
29
+ * Get byte range starting from the specified offset
30
+ *
31
+ * @example
32
+ * ```
33
+ * GET /file.parquet HTTP/1.1
34
+ * Range: bytes=1024-
35
+ * ```
36
+ */
37
+ type: 'offset';
38
+ /** Start byte position (inclusive) */
39
+ offset: number;
40
+ }
41
+ | {
42
+ /**
43
+ * Get byte range starting from the specified suffix
44
+ *
45
+ * @example
46
+ * ```
47
+ * GET /file.parquet HTTP/1.1
48
+ * Range: bytes=-1024
49
+ * ```
50
+ */
51
+ type: 'suffix';
52
+ /** End byte position (inclusive) */
53
+ suffix: number;
54
+ };
55
+
56
+ /** HTTP method passed to object store */
57
+ export type HttpMethod = 'GET' | 'HEAD';
36
58
 
59
+ /** HTTP response from object store */
60
+ export type ObjectStoreResponse<Method extends HttpMethod = HttpMethod> =
61
+ | {
62
+ /** Will be translated to 500 Internal Server Error by the handler */
63
+ type: 'InternalError';
64
+ }
65
+ | {
66
+ /** Will be translated to 404 Not Found by the handler */
67
+ type: 'NotFound';
68
+ }
69
+ | {
70
+ /** Will be translated to 416 Range Not Satisfiable by the handler */
71
+ type: 'RangeNotSatisfiable';
72
+ /** Total file size in bytes */
73
+ size: number;
74
+ }
75
+ | {
76
+ /** Will be translated to 200 OK or 206 Partial Content by the handler */
77
+ type: 'Ok';
78
+ /** Total file size in bytes */
79
+ size: number;
80
+ /** Stream of file content, undefined for HEAD requests */
81
+ data: Method extends 'HEAD' ? undefined : Readable;
82
+ }
83
+
84
+ /** Common options for object store creation */
85
+ export interface ObjectStoreOptions {
86
+ /** Logger instance, no logging is performed when not provided */
87
+ logger?: Logger;
88
+ }
89
+
90
+ /** Options for file system object store creation */
91
+ export interface FsStoreOptions extends ObjectStoreOptions {
92
+ /** Local directory to serve files from */
93
+ rootDir: string;
94
+ }
95
+
96
+ /** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */
97
+ export interface ObjectStore {
37
98
  /**
38
- * Execute action with readable stream (actions can be concurrency limited by the store).
39
- * Action resolves when stream is closed by handler @see HttpHelpers.createRequestHandler
40
- *
41
- * @param filename - existing file name (for which @see ObjectStore.getFileSize returned non-negative value)
42
- * @param range - valid range of bytes to read from the file (store may skip validation)
43
- * @param action - function to execute with the stream, responsible for closing the stream
44
- * @returns promise that resolves after the action is completed
45
- *
46
- * @example
47
- * ```ts
48
- * async withReadStream(params: {
49
- * filename: string;
50
- * range: FileRange;
51
- * action: (stream: Readable) => Promise<void>;
52
- * }): Promise<void> {
53
- * const { filename, range, action } = params;
54
- * const filePath = this.resolve(filename);
55
- *
56
- * try {
57
- * const stream = createReadStream(filePath, range);
58
- * return await action(stream);
59
- * } catch (err: unknown) {
60
- * console.error(`failed to create read stream for ${filename} - ${ensureError(err)}`);
61
- * throw;
62
- * }
63
- * }
64
- * ```
99
+ * Proxy HTTP(S) request for parquet file to object store.
100
+ * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler
101
+ * Callback API is used so that ObjectStore can limit the number of concurrent requests.
65
102
  */
66
- withReadStream(params: {
67
- filename: string;
68
- range: FileRange;
69
- action: (stream: Readable) => Promise<void>;
70
- }): Promise<void>;
103
+ request<Method extends HttpMethod>(
104
+ filename: ParquetFileName,
105
+ params: {
106
+ method: Method;
107
+ range?: HttpRange;
108
+ signal?: AbortSignal;
109
+ callback: (response: ObjectStoreResponse<Method>) => Promise<void>;
110
+ }
111
+ ): void;
71
112
  }
72
113
 
73
114
  /** Object store base URL in format accepted by Apache DataFusion and DuckDB */
@@ -136,7 +177,7 @@ export interface HttpHelpers {
136
177
  * Create an object store for serving files from a local directory.
137
178
  * Rejects if the provided path does not exist or is not a directory.
138
179
  */
139
- createFsStore(rootDir: string): Promise<ObjectStore>;
180
+ createFsStore(options: FsStoreOptions): Promise<ObjectStore>;
140
181
 
141
182
  /**
142
183
  * Create an HTTP request handler for serving files from an object store.
@@ -153,7 +194,7 @@ export interface HttpHelpers {
153
194
  * ```ts
154
195
  * const rootDir = '/path/to/directory/with/parquet/files';
155
196
  *
156
- * let store = await HttpHelpers.createFsStore(rootDir).catch((err: unknown) => {
197
+ * let store = await HttpHelpers.createFsStore({ rootDir }).catch((err: unknown) => {
157
198
  * throw new Error(`Failed to create file store for ${rootDir} - ${ensureError(err)}`);
158
199
  * });
159
200
  *
package/src/pframe/internal_api/pframe.ts CHANGED
@@ -1,10 +1,6 @@
1
1
  import type { PFrameFactoryAPIV2, PFrameFactoryAPIV3 } from './api_factory';
2
2
  import type { PFrameReadAPIV8, PFrameReadAPIV9 } from './api_read';
3
-
4
- export type Logger = (
5
- level: 'info' | 'warn' | 'error',
6
- message: string
7
- ) => void;
3
+ import type { Logger } from './common';
8
4
 
9
5
  export interface PFrameV9 extends PFrameFactoryAPIV2, PFrameReadAPIV8 {}
10
6