@milaboratories/pl-model-middle-layer 1.8.19 → 1.8.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pframe/internal_api/api_factory.d.ts +2 -2
- package/dist/pframe/internal_api/api_factory.d.ts.map +1 -1
- package/dist/pframe/internal_api/http_helpers.cjs.map +1 -1
- package/dist/pframe/internal_api/http_helpers.d.ts +8 -5
- package/dist/pframe/internal_api/http_helpers.d.ts.map +1 -1
- package/dist/pframe/internal_api/http_helpers.js.map +1 -1
- package/package.json +3 -3
- package/src/pframe/internal_api/api_factory.ts +2 -2
- package/src/pframe/internal_api/http_helpers.ts +8 -5
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { BinaryPartitionedDataInfo, JsonDataInfo, JsonPartitionedDataInfo, ParquetChunk, ParquetPartitionedDataInfo, PColumnSpec, PObjectId } from '@milaboratories/pl-model-common';
|
|
2
|
-
import {
|
|
2
|
+
import { HttpServerInfo } from './http_helpers';
|
|
3
3
|
/** Abstract identifier of a data blob that can be requested from the storage backend */
|
|
4
4
|
export type PFrameBlobId = string;
|
|
5
5
|
/** Path of the file containing requested data (blob). This path is returned by
|
|
@@ -19,7 +19,7 @@ export interface PFrameDataSourceV2 {
|
|
|
19
19
|
* Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer}
|
|
20
20
|
* When not provided, parquet BlobIds would be treated as local file paths.
|
|
21
21
|
*/
|
|
22
|
-
parquetServer?:
|
|
22
|
+
parquetServer?: HttpServerInfo;
|
|
23
23
|
}
|
|
24
24
|
/**
|
|
25
25
|
* Union type representing all possible data storage formats for PColumn data.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api_factory.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,yBAAyB,EACzB,YAAY,EACZ,uBAAuB,EACvB,YAAY,EACZ,0BAA0B,EAC1B,WAAW,EACX,SAAS,EACV,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEL,
|
|
1
|
+
{"version":3,"file":"api_factory.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,yBAAyB,EACzB,YAAY,EACZ,uBAAuB,EACvB,YAAY,EACZ,0BAA0B,EAC1B,WAAW,EACX,SAAS,EACV,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEL,cAAc,EACf,MAAM,gBAAgB,CAAC;AAExB,wFAAwF;AACxF,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAGlC;+EAC+E;AAC/E,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,gGAAgG;AAChG,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpD,qEAAqE;IACrE,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAE9D;;;OAGG;IACH,aAAa,CAAC,EAAE,cAAc,CAAC;CAChC;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,CAAC,IAAI,IACrB,YAAY,GACZ,uBAAuB,CAAC,IAAI,CAAC,GAC7B,yBAAyB,CAAC,IAAI,CAAC,GAC/B,0BAA0B,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;AAEnD;oBACoB;AACpB,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD,8CAA8C;IAC9C,aAAa,CAAC,UAAU,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAEpD,qCAAqC;IACrC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI,CAAC;IAElE;;;OAGG;IACH,aAAa,CACX,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,QAAQ,CAAC,YAAY,CAAC,EAChC,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;wEACoE;IACpE,OAAO,IAAI,IAAI,CAAC;CACjB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http_helpers.cjs","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n
|
|
1
|
+
{"version":3,"file":"http_helpers.cjs","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n /**\n * Will be translated to 500 Internal Server Error by the handler\n * or 408 Request Timeout if the request was aborted\n */\n type: 'InternalError';\n }\n | {\n /** Will be translated to 404 Not Found by the handler */\n type: 'NotFound';\n }\n | {\n /** Will be translated to 416 Range Not Satisfiable by the handler */\n type: 'RangeNotSatisfiable';\n /** Total file size in bytes */\n size: number;\n }\n | {\n /** Will be translated to 200 OK or 206 Partial Content by the handler */\n type: 'Ok';\n /** Total file size in bytes */\n size: number;\n /** File range translated from HTTP range */\n range: FileRange;\n /** Stream of file content, undefined for HEAD requests */\n data?: Readable;\n }\n\n/** Common options for object store creation */\nexport interface ObjectStoreOptions {\n /** Logger instance, no logging is performed when not provided */\n logger?: Logger;\n}\n\n/** Options for file system object store creation */\nexport interface FsStoreOptions extends ObjectStoreOptions {\n /** Local directory to serve files from */\n rootDir: string;\n}\n\n/** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */\nexport abstract class ObjectStore {\n protected readonly logger: Logger;\n\n constructor(options: ObjectStoreOptions) {\n this.logger = options.logger ?? (() => {});\n }\n\n /** Translate HTTP range to file range, @returns null if the range is not satisfiable */\n protected translate(fileSize: number, range?: HttpRange): FileRange | null {\n if (!range) return { start: 0, end: fileSize - 1 };\n switch (range.type) {\n case 'bounded':\n if (range.end >= fileSize) return null;\n return { start: range.start, end: range.end };\n case 'offset':\n if (range.offset >= fileSize) return null;\n return { start: range.offset, end: fileSize - 1 };\n case 'suffix':\n if (range.suffix > fileSize) return null;\n return { start: fileSize - range.suffix, end: fileSize - 1 };\n }\n }\n\n /**\n * Proxy HTTP(S) request for parquet file to object store.\n * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler\n * Callback API is used so that ObjectStore can limit the number of concurrent requests.\n */\n abstract request(\n filename: ParquetFileName,\n params: {\n method: HttpMethod;\n range?: HttpRange;\n signal: AbortSignal;\n callback: (response: ObjectStoreResponse) => Promise<void>;\n }\n ): void;\n}\n\n/** Object store base URL in format accepted by Apache DataFusion and DuckDB */\nexport type ObjectStoreUrl = Branded<string, 'PFrameInternal.ObjectStoreUrl'>;\n\n/** HTTP(S) request handler creation options */\nexport type RequestHandlerOptions = {\n /** Object store to serve files from, @see HttpHelpers.createFsStore */\n store: ObjectStore;\n /** Here will go caching options... */\n}\n\n/** Server configuration options */\nexport type HttpServerOptions = {\n /** HTTP(S) request handler function, @see HttpHelpers.createRequestHandler */\n handler: RequestListener;\n /** Port to bind to, @default 0 for auto-assignment */\n port?: number;\n /** Do not apply authorization middleware to @param handler */\n noAuth?: true;\n /** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */\n noHttps?: true;\n};\n\n/**\n * Long unique opaque string for use in Bearer authorization header\n * \n * @example\n * ```ts\n * request.setHeader('Authorization', `Bearer ${authToken}`);\n * ```\n */\nexport type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthorizationToken'>;\n\n/**\n * TLS certificate in PEM format\n * \n * @example\n * ```txt\n * -----BEGIN CERTIFICATE-----\n * MIIC2zCCAcOgAwIBAgIJaVW7...\n * ...\n * ...Yf9CRK8fgnukKM7TJ\n * -----END CERTIFICATE-----\n * ```\n */\nexport type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;\n\n/** HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */\nexport type HttpServerInfo = {\n /** URL of the HTTP(S) server formatted as `http{s}://<host>:<port>/` */\n url: ObjectStoreUrl;\n /** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */\n authToken?: HttpAuthorizationToken;\n /** Encoded CA certificate of HTTPS server, undefined when @see HttpServerOptions.noHttps flag is set */\n encodedCaCert?: Base64Encoded<PemCertificate>;\n};\n\n/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */\nexport interface HttpServer {\n /** Server configuration information for initiating connections from clients */\n get info(): HttpServerInfo;\n /** Promise that resolves when the server is stopped */\n get stopped(): Promise<void>;\n /** Request server stop, returns the same promise as @see HttpServer.stopped */\n stop(): Promise<void>;\n}\n\n/** List of HTTP(S) related helper functions exposed by PFrame module */\nexport interface HttpHelpers {\n /**\n * Create an object store for serving files from a local directory.\n * Rejects if the provided path does not exist or is not a directory.\n */\n createFsStore(options: FsStoreOptions): Promise<ObjectStore>;\n\n /**\n * Create an HTTP request handler for serving files from an object store.\n * Accepts only paths of the form `/<filename>.parquet`, returns 410 otherwise.\n * Assumes that files are immutable (and sets cache headers accordingly).\n */\n createRequestHandler(options: RequestHandlerOptions): RequestListener;\n\n /**\n * Serve HTTP(S) requests using the provided handler on localhost port.\n * @returns promise that resolves when the server has stopped.\n *\n * @example\n * ```ts\n * const rootDir = '/path/to/directory/with/parquet/files';\n *\n * let store = await HttpHelpers.createFsStore({ rootDir }).catch((err: unknown) => {\n * throw new Error(`Failed to create file store for ${rootDir} - ${ensureError(err)}`);\n * });\n *\n * const server = await HttpHelpers.createHttpServer({\n * handler: HttpHelpers.createRequestHandler({ store }),\n * }).catch((err: unknown) => {\n * throw new Error(`Failed to start HTTPS server - ${ensureError(err)}`);\n * });\n *\n * const { url, authToken, encodedCaCert } = server.info;\n *\n * await server.stop();\n * ```\n */\n createHttpServer(options: HttpServerOptions): Promise<HttpServer>;\n}\n"],"names":[],"mappings":";;AA2GA;MACsB,WAAW,CAAA;AACZ,IAAA,MAAM;AAEzB,IAAA,WAAA,CAAY,OAA2B,EAAA;AACrC,QAAA,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,MAAK,EAAE,CAAC,CAAC;IAC5C;;IAGU,SAAS,CAAC,QAAgB,EAAE,KAAiB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AAClD,QAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,YAAA,KAAK,SAAS;AACZ,gBAAA,IAAI,KAAK,CAAC,GAAG,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACtC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE;AAC/C,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACzC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AACnD,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACxC,gBAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;;IAElE;AAgBD;;;;"}
|
|
@@ -57,7 +57,10 @@ export type HttpRange = {
|
|
|
57
57
|
export type HttpMethod = 'GET' | 'HEAD';
|
|
58
58
|
/** HTTP response from object store */
|
|
59
59
|
export type ObjectStoreResponse = {
|
|
60
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Will be translated to 500 Internal Server Error by the handler
|
|
62
|
+
* or 408 Request Timeout if the request was aborted
|
|
63
|
+
*/
|
|
61
64
|
type: 'InternalError';
|
|
62
65
|
} | {
|
|
63
66
|
/** Will be translated to 404 Not Found by the handler */
|
|
@@ -145,9 +148,9 @@ export type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthori
|
|
|
145
148
|
* ```
|
|
146
149
|
*/
|
|
147
150
|
export type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;
|
|
148
|
-
/**
|
|
149
|
-
export type
|
|
150
|
-
/** URL of the
|
|
151
|
+
/** HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
|
|
152
|
+
export type HttpServerInfo = {
|
|
153
|
+
/** URL of the HTTP(S) server formatted as `http{s}://<host>:<port>/` */
|
|
151
154
|
url: ObjectStoreUrl;
|
|
152
155
|
/** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */
|
|
153
156
|
authToken?: HttpAuthorizationToken;
|
|
@@ -157,7 +160,7 @@ export type ParquetServerInfo = {
|
|
|
157
160
|
/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */
|
|
158
161
|
export interface HttpServer {
|
|
159
162
|
/** Server configuration information for initiating connections from clients */
|
|
160
|
-
get info():
|
|
163
|
+
get info(): HttpServerInfo;
|
|
161
164
|
/** Promise that resolves when the server is stopped */
|
|
162
165
|
get stopped(): Promise<void>;
|
|
163
166
|
/** Request server stop, returns the same promise as @see HttpServer.stopped */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http_helpers.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAC9E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,wBAAwB;AACxB,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE,gCAAgC,CAAC,CAAC;AAE7F,MAAM,MAAM,SAAS,GAAG;IACtB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,CAAA;AAED,2FAA2F;AAC3F,MAAM,MAAM,SAAS,GACjB;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,SAAS,CAAC;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,yCAAyC;AACzC,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,CAAC;AAExC,sCAAsC;AACtC,MAAM,MAAM,mBAAmB,GAC3B;IACE
|
|
1
|
+
{"version":3,"file":"http_helpers.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAC9E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,wBAAwB;AACxB,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE,gCAAgC,CAAC,CAAC;AAE7F,MAAM,MAAM,SAAS,GAAG;IACtB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,CAAA;AAED,2FAA2F;AAC3F,MAAM,MAAM,SAAS,GACjB;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,SAAS,CAAC;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,yCAAyC;AACzC,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,CAAC;AAExC,sCAAsC;AACtC,MAAM,MAAM,mBAAmB,GAC3B;IACE;;;OAGG;IACH,IAAI,EAAE,eAAe,CAAC;CACvB,GACD;IACE,yDAAyD;IACzD,IAAI,EAAE,UAAU,CAAC;CAClB,GACD;IACE,qEAAqE;IACrE,IAAI,EAAE,qBAAqB,CAAC;IAC5B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,yEAAyE;IACzE,IAAI,EAAE,IAAI,CAAC;IACX,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,KAAK,EAAE,SAAS,CAAC;IACjB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,QAAQ,CAAC;CACjB,CAAA;AAEL,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,oDAAoD;AACpD,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iGAAiG;AACjG,8BAAsB,WAAW;IAC/B,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEtB,OAAO,EAAE,kBAAkB;IAIvC,wFAAwF;IACxF,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,IAAI;IAe1E;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CACd,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE;QACN,MAAM,EAAE,UAAU,CAAC;QACnB,KAAK,CAAC,EAAE,SAAS,CAAC;QAClB,MAAM,EAAE,WAAW,CAAC;QACpB,QAAQ,EAAE,CAAC,QAAQ,EAAE,mBAAmB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;KAC5D,GACA,IAAI;CACR;AAED,+EAA+E;AAC/E,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,+CAA+C;AAC/C,MAAM,MAAM,qBAAqB,GAAG;IAClC,uEAAuE;IACvE,KAAK,EAAE,WAAW,CAAC;CAEpB,CAAA;AAED,mCAAmC;AACnC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,8EAA8E;IAC9E,OAAO,EAAE,eAAe,CAAC;IACzB,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,kFAAkF;IAClF,OAAO,CAAC,EAAE,IAAI,CAAC;CAChB,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,EAAE,uCAAuC,CAAC,CAAC;AAE9F;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,+EAA+E;AAC/E,MAAM,MAAM,cAAc,GAAG;IAC3B,wEAAwE;IACxE,GAAG,EAAE,cAAc,CAAC;IACpB,sGAAsG;IACtG,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,wGAAwG;IACxG,aAAa,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CAC/C,CAAC;AAEF,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,+EAA+E;IAC/E,IAAI,IAAI,IAAI,cAAc,CAAC;IAC3B,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,+EAA+E;IAC/E,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE7D;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,eAAe,CAAC;IAEtE;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACnE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http_helpers.js","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n
|
|
1
|
+
{"version":3,"file":"http_helpers.js","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n /**\n * Will be translated to 500 Internal Server Error by the handler\n * or 408 Request Timeout if the request was aborted\n */\n type: 'InternalError';\n }\n | {\n /** Will be translated to 404 Not Found by the handler */\n type: 'NotFound';\n }\n | {\n /** Will be translated to 416 Range Not Satisfiable by the handler */\n type: 'RangeNotSatisfiable';\n /** Total file size in bytes */\n size: number;\n }\n | {\n /** Will be translated to 200 OK or 206 Partial Content by the handler */\n type: 'Ok';\n /** Total file size in bytes */\n size: number;\n /** File range translated from HTTP range */\n range: FileRange;\n /** Stream of file content, undefined for HEAD requests */\n data?: Readable;\n }\n\n/** Common options for object store creation */\nexport interface ObjectStoreOptions {\n /** Logger instance, no logging is performed when not provided */\n logger?: Logger;\n}\n\n/** Options for file system object store creation */\nexport interface FsStoreOptions extends ObjectStoreOptions {\n /** Local directory to serve files from */\n rootDir: string;\n}\n\n/** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */\nexport abstract class ObjectStore {\n protected readonly logger: Logger;\n\n constructor(options: ObjectStoreOptions) {\n this.logger = options.logger ?? (() => {});\n }\n\n /** Translate HTTP range to file range, @returns null if the range is not satisfiable */\n protected translate(fileSize: number, range?: HttpRange): FileRange | null {\n if (!range) return { start: 0, end: fileSize - 1 };\n switch (range.type) {\n case 'bounded':\n if (range.end >= fileSize) return null;\n return { start: range.start, end: range.end };\n case 'offset':\n if (range.offset >= fileSize) return null;\n return { start: range.offset, end: fileSize - 1 };\n case 'suffix':\n if (range.suffix > fileSize) return null;\n return { start: fileSize - range.suffix, end: fileSize - 1 };\n }\n }\n\n /**\n * Proxy HTTP(S) request for parquet file to object store.\n * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler\n * Callback API is used so that ObjectStore can limit the number of concurrent requests.\n */\n abstract request(\n filename: ParquetFileName,\n params: {\n method: HttpMethod;\n range?: HttpRange;\n signal: AbortSignal;\n callback: (response: ObjectStoreResponse) => Promise<void>;\n }\n ): void;\n}\n\n/** Object store base URL in format accepted by Apache DataFusion and DuckDB */\nexport type ObjectStoreUrl = Branded<string, 'PFrameInternal.ObjectStoreUrl'>;\n\n/** HTTP(S) request handler creation options */\nexport type RequestHandlerOptions = {\n /** Object store to serve files from, @see HttpHelpers.createFsStore */\n store: ObjectStore;\n /** Here will go caching options... */\n}\n\n/** Server configuration options */\nexport type HttpServerOptions = {\n /** HTTP(S) request handler function, @see HttpHelpers.createRequestHandler */\n handler: RequestListener;\n /** Port to bind to, @default 0 for auto-assignment */\n port?: number;\n /** Do not apply authorization middleware to @param handler */\n noAuth?: true;\n /** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */\n noHttps?: true;\n};\n\n/**\n * Long unique opaque string for use in Bearer authorization header\n * \n * @example\n * ```ts\n * request.setHeader('Authorization', `Bearer ${authToken}`);\n * ```\n */\nexport type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthorizationToken'>;\n\n/**\n * TLS certificate in PEM format\n * \n * @example\n * ```txt\n * -----BEGIN CERTIFICATE-----\n * MIIC2zCCAcOgAwIBAgIJaVW7...\n * ...\n * ...Yf9CRK8fgnukKM7TJ\n * -----END CERTIFICATE-----\n * ```\n */\nexport type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;\n\n/** HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */\nexport type HttpServerInfo = {\n /** URL of the HTTP(S) server formatted as `http{s}://<host>:<port>/` */\n url: ObjectStoreUrl;\n /** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */\n authToken?: HttpAuthorizationToken;\n /** Encoded CA certificate of HTTPS server, undefined when @see HttpServerOptions.noHttps flag is set */\n encodedCaCert?: Base64Encoded<PemCertificate>;\n};\n\n/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */\nexport interface HttpServer {\n /** Server configuration information for initiating connections from clients */\n get info(): HttpServerInfo;\n /** Promise that resolves when the server is stopped */\n get stopped(): Promise<void>;\n /** Request server stop, returns the same promise as @see HttpServer.stopped */\n stop(): Promise<void>;\n}\n\n/** List of HTTP(S) related helper functions exposed by PFrame module */\nexport interface HttpHelpers {\n /**\n * Create an object store for serving files from a local directory.\n * Rejects if the provided path does not exist or is not a directory.\n */\n createFsStore(options: FsStoreOptions): Promise<ObjectStore>;\n\n /**\n * Create an HTTP request handler for serving files from an object store.\n * Accepts only paths of the form `/<filename>.parquet`, returns 410 otherwise.\n * Assumes that files are immutable (and sets cache headers accordingly).\n */\n createRequestHandler(options: RequestHandlerOptions): RequestListener;\n\n /**\n * Serve HTTP(S) requests using the provided handler on localhost port.\n * @returns promise that resolves when the server has stopped.\n *\n * @example\n * ```ts\n * const rootDir = '/path/to/directory/with/parquet/files';\n *\n * let store = await HttpHelpers.createFsStore({ rootDir }).catch((err: unknown) => {\n * throw new Error(`Failed to create file store for ${rootDir} - ${ensureError(err)}`);\n * });\n *\n * const server = await HttpHelpers.createHttpServer({\n * handler: HttpHelpers.createRequestHandler({ store }),\n * }).catch((err: unknown) => {\n * throw new Error(`Failed to start HTTPS server - ${ensureError(err)}`);\n * });\n *\n * const { url, authToken, encodedCaCert } = server.info;\n *\n * await server.stop();\n * ```\n */\n createHttpServer(options: HttpServerOptions): Promise<HttpServer>;\n}\n"],"names":[],"mappings":"AA2GA;MACsB,WAAW,CAAA;AACZ,IAAA,MAAM;AAEzB,IAAA,WAAA,CAAY,OAA2B,EAAA;AACrC,QAAA,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,MAAK,EAAE,CAAC,CAAC;IAC5C;;IAGU,SAAS,CAAC,QAAgB,EAAE,KAAiB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AAClD,QAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,YAAA,KAAK,SAAS;AACZ,gBAAA,IAAI,KAAK,CAAC,GAAG,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACtC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE;AAC/C,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACzC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AACnD,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACxC,gBAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;;IAElE;AAgBD;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-model-middle-layer",
|
|
3
|
-
"version": "1.8.
|
|
3
|
+
"version": "1.8.21",
|
|
4
4
|
"description": "Common model between middle layer and non-block UI code",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -24,9 +24,9 @@
|
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"typescript": "~5.6.3",
|
|
27
|
-
"@milaboratories/
|
|
27
|
+
"@milaboratories/build-configs": "1.0.8",
|
|
28
28
|
"@milaboratories/ts-configs": "1.0.6",
|
|
29
|
-
"@milaboratories/
|
|
29
|
+
"@milaboratories/ts-builder": "1.0.5"
|
|
30
30
|
},
|
|
31
31
|
"scripts": {
|
|
32
32
|
"type-check": "ts-builder types --target node",
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
} from '@milaboratories/pl-model-common';
|
|
11
11
|
import {
|
|
12
12
|
HttpHelpers,
|
|
13
|
-
|
|
13
|
+
HttpServerInfo,
|
|
14
14
|
} from './http_helpers';
|
|
15
15
|
|
|
16
16
|
/** Abstract identifier of a data blob that can be requested from the storage backend */
|
|
@@ -37,7 +37,7 @@ export interface PFrameDataSourceV2 {
|
|
|
37
37
|
* Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer}
|
|
38
38
|
* When not provided, parquet BlobIds would be treated as local file paths.
|
|
39
39
|
*/
|
|
40
|
-
parquetServer?:
|
|
40
|
+
parquetServer?: HttpServerInfo;
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
/**
|
|
@@ -66,7 +66,10 @@ export type HttpMethod = 'GET' | 'HEAD';
|
|
|
66
66
|
/** HTTP response from object store */
|
|
67
67
|
export type ObjectStoreResponse =
|
|
68
68
|
| {
|
|
69
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* Will be translated to 500 Internal Server Error by the handler
|
|
71
|
+
* or 408 Request Timeout if the request was aborted
|
|
72
|
+
*/
|
|
70
73
|
type: 'InternalError';
|
|
71
74
|
}
|
|
72
75
|
| {
|
|
@@ -188,9 +191,9 @@ export type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthori
|
|
|
188
191
|
*/
|
|
189
192
|
export type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;
|
|
190
193
|
|
|
191
|
-
/**
|
|
192
|
-
export type
|
|
193
|
-
/** URL of the
|
|
194
|
+
/** HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
|
|
195
|
+
export type HttpServerInfo = {
|
|
196
|
+
/** URL of the HTTP(S) server formatted as `http{s}://<host>:<port>/` */
|
|
194
197
|
url: ObjectStoreUrl;
|
|
195
198
|
/** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */
|
|
196
199
|
authToken?: HttpAuthorizationToken;
|
|
@@ -201,7 +204,7 @@ export type ParquetServerInfo = {
|
|
|
201
204
|
/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */
|
|
202
205
|
export interface HttpServer {
|
|
203
206
|
/** Server configuration information for initiating connections from clients */
|
|
204
|
-
get info():
|
|
207
|
+
get info(): HttpServerInfo;
|
|
205
208
|
/** Promise that resolves when the server is stopped */
|
|
206
209
|
get stopped(): Promise<void>;
|
|
207
210
|
/** Request server stop, returns the same promise as @see HttpServer.stopped */
|