@milaboratories/pl-model-middle-layer 1.8.17 → 1.8.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pframe/internal_api/api_factory.d.ts +3 -41
- package/dist/pframe/internal_api/api_factory.d.ts.map +1 -1
- package/dist/pframe/internal_api/http_helpers.cjs.map +1 -1
- package/dist/pframe/internal_api/http_helpers.d.ts +15 -10
- package/dist/pframe/internal_api/http_helpers.d.ts.map +1 -1
- package/dist/pframe/internal_api/http_helpers.js.map +1 -1
- package/dist/pframe/internal_api/pframe.d.ts +1 -19
- package/dist/pframe/internal_api/pframe.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/pframe/internal_api/api_factory.ts +2 -55
- package/src/pframe/internal_api/http_helpers.ts +16 -10
- package/src/pframe/internal_api/pframe.ts +1 -21
|
@@ -1,31 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { BinaryPartitionedDataInfo, JsonDataInfo, JsonPartitionedDataInfo, ParquetChunk, ParquetPartitionedDataInfo, PColumnSpec, PObjectId } from '@milaboratories/pl-model-common';
|
|
2
|
+
import { ParquetServerInfo } from './http_helpers';
|
|
3
3
|
/** Abstract identifier of a data blob that can be requested from the storage backend */
|
|
4
4
|
export type PFrameBlobId = string;
|
|
5
5
|
/** Path of the file containing requested data (blob). This path is returned by
|
|
6
6
|
* {@link BlobPathResolver} as soon as blob materialized in the file system. */
|
|
7
7
|
export type FilePath = string;
|
|
8
8
|
/** Data source allows PFrame to retrieve the data blobs for columns with assigned data info. */
|
|
9
|
-
export type PFrameDataSource = {
|
|
10
|
-
/**
|
|
11
|
-
* PFrame may notify storage backend about its plans to use particular blobs in the future.
|
|
12
|
-
* Storage backend will do its best to preload specified blob so the subsequent
|
|
13
|
-
* {@link resolveBlob} will quickly return preloaded file path.
|
|
14
|
-
*/
|
|
15
|
-
preloadBlob(blobIds: PFrameBlobId[]): Promise<void>;
|
|
16
|
-
/** Returns raw blob data given the blob id from {@link DataInfo}. */
|
|
17
|
-
resolveBlobContent(blobId: PFrameBlobId): Promise<Uint8Array>;
|
|
18
|
-
};
|
|
19
|
-
/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
|
|
20
|
-
export type ParquetServerConfig = {
|
|
21
|
-
/** URL of the parquet HTTP(S) server */
|
|
22
|
-
url: ObjectStoreUrl;
|
|
23
|
-
/** Authorization token for Bearer scheme */
|
|
24
|
-
authToken?: HttpAuthorizationToken;
|
|
25
|
-
/** CA certificate of HTTPS server */
|
|
26
|
-
caCert?: Base64Encoded<PemCertificate>;
|
|
27
|
-
};
|
|
28
|
-
/** Data source allows PFrame to retrieve the data blobs for columns with assigned data info. */
|
|
29
9
|
export interface PFrameDataSourceV2 {
|
|
30
10
|
/**
|
|
31
11
|
* PFrame may notify storage backend about its plans to use particular blobs in the future.
|
|
@@ -39,7 +19,7 @@ export interface PFrameDataSourceV2 {
|
|
|
39
19
|
* Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer}
|
|
40
20
|
* When not provided, parquet BlobIds would be treated as local file paths.
|
|
41
21
|
*/
|
|
42
|
-
parquetServer?:
|
|
22
|
+
parquetServer?: ParquetServerInfo;
|
|
43
23
|
}
|
|
44
24
|
/**
|
|
45
25
|
* Union type representing all possible data storage formats for PColumn data.
|
|
@@ -48,24 +28,6 @@ export interface PFrameDataSourceV2 {
|
|
|
48
28
|
* @template Blob - Type parameter representing the storage reference type (could be ResourceInfo, PFrameBlobId, etc.)
|
|
49
29
|
*/
|
|
50
30
|
export type DataInfo<Blob> = JsonDataInfo | JsonPartitionedDataInfo<Blob> | BinaryPartitionedDataInfo<Blob> | ParquetPartitionedDataInfo<ParquetChunk<Blob>>;
|
|
51
|
-
/** API exposed by PFrames library allowing to create and provide data for
|
|
52
|
-
* PFrame objects */
|
|
53
|
-
export interface PFrameFactoryAPIV3 extends Disposable {
|
|
54
|
-
/** Associates data source with this PFrame */
|
|
55
|
-
setDataSource(dataSource: PFrameDataSource): void;
|
|
56
|
-
/** Adds PColumn without data info */
|
|
57
|
-
addColumnSpec(columnId: PObjectId, columnSpec: PColumnSpec): void;
|
|
58
|
-
/**
|
|
59
|
-
* Assign data info to the specified PColumn.
|
|
60
|
-
* For parquet data info, schema resolution via network is performed during this call.
|
|
61
|
-
*/
|
|
62
|
-
setColumnData(columnId: PObjectId, dataInfo: DataInfo<PFrameBlobId>, options?: {
|
|
63
|
-
signal?: AbortSignal;
|
|
64
|
-
}): Promise<void>;
|
|
65
|
-
/** Releases all the data previously added to PFrame using methods above,
|
|
66
|
-
* any interactions with disposed PFrame will result in exception */
|
|
67
|
-
dispose(): void;
|
|
68
|
-
}
|
|
69
31
|
/** API exposed by PFrames library allowing to create and provide data for
|
|
70
32
|
* PFrame objects */
|
|
71
33
|
export interface PFrameFactoryAPIV4 extends Disposable {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api_factory.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_factory.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"api_factory.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,yBAAyB,EACzB,YAAY,EACZ,uBAAuB,EACvB,YAAY,EACZ,0BAA0B,EAC1B,WAAW,EACX,SAAS,EACV,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEL,iBAAiB,EAClB,MAAM,gBAAgB,CAAC;AAExB,wFAAwF;AACxF,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAGlC;+EAC+E;AAC/E,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,gGAAgG;AAChG,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpD,qEAAqE;IACrE,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAE9D;;;OAGG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;CACnC;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,CAAC,IAAI,IACrB,YAAY,GACZ,uBAAuB,CAAC,IAAI,CAAC,GAC7B,yBAAyB,CAAC,IAAI,CAAC,GAC/B,0BAA0B,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;AAEnD;oBACoB;AACpB,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD,8CAA8C;IAC9C,aAAa,CAAC,UAAU,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAEpD,qCAAqC;IACrC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI,CAAC;IAElE;;;OAGG;IACH,aAAa,CACX,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,QAAQ,CAAC,YAAY,CAAC,EAChC,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;wEACoE;IACpE,OAAO,IAAI,IAAI,CAAC;CACjB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http_helpers.cjs","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n /** Will be translated to 500 Internal Server Error by the handler */\n type: 'InternalError';\n }\n | {\n /** Will be translated to 404 Not Found by the handler */\n type: 'NotFound';\n }\n | {\n /** Will be translated to 416 Range Not Satisfiable by the handler */\n type: 'RangeNotSatisfiable';\n /** Total file size in bytes */\n size: number;\n }\n | {\n /** Will be translated to 200 OK or 206 Partial Content by the handler */\n type: 'Ok';\n /** Total file size in bytes */\n size: number;\n /** File range translated from HTTP range */\n range: FileRange;\n /** Stream of file content, undefined for HEAD requests */\n data?: Readable;\n }\n\n/** Common options for object store creation */\nexport interface ObjectStoreOptions {\n /** Logger instance, no logging is performed when not provided */\n logger?: Logger;\n}\n\n/** Options for file system object store creation */\nexport interface FsStoreOptions extends ObjectStoreOptions {\n /** Local directory to serve files from */\n rootDir: string;\n}\n\n/** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */\nexport abstract class ObjectStore {\n protected readonly logger: Logger;\n\n constructor(options: ObjectStoreOptions) {\n this.logger = options.logger ?? (() => {});\n }\n\n /** Translate HTTP range to file range, @returns null if the range is not satisfiable */\n protected translate(fileSize: number, range?: HttpRange): FileRange | null {\n if (!range) return { start: 0, end: fileSize - 1 };\n switch (range.type) {\n case 'bounded':\n if (range.end >= fileSize) return null;\n return { start: range.start, end: range.end };\n case 'offset':\n if (range.offset >= fileSize) return null;\n return { start: range.offset, end: fileSize - 1 };\n case 'suffix':\n if (range.suffix > fileSize) return null;\n return { start: fileSize - range.suffix, end: fileSize - 1 };\n }\n }\n\n /**\n * Proxy HTTP(S) request for parquet file to object store.\n * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler\n * Callback API is used so that ObjectStore can limit the number of concurrent requests.\n */\n abstract request(\n filename: ParquetFileName,\n params: {\n method: HttpMethod;\n range?: HttpRange;\n signal: AbortSignal;\n callback: (response: ObjectStoreResponse) => Promise<void>;\n }\n ): void;\n}\n\n/** Object store base URL in format accepted by Apache DataFusion and DuckDB */\nexport type ObjectStoreUrl = Branded<string, 'PFrameInternal.ObjectStoreUrl'>;\n\n/** HTTP(S) request handler creation options */\nexport type RequestHandlerOptions = {\n /** Object store to serve files from, @see HttpHelpers.createFsStore */\n store: ObjectStore;\n /** Here will go caching options... */\n}\n\n/** Server configuration options */\nexport type HttpServerOptions = {\n /** HTTP(S) request handler function, @see HttpHelpers.createRequestHandler */\n handler: RequestListener;\n /** Port to bind to, @default 0 for auto-assignment */\n port?: number;\n /** Do not apply authorization middleware to @param handler */\n noAuth?: true;\n /** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */\n
|
|
1
|
+
{"version":3,"file":"http_helpers.cjs","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n /** Will be translated to 500 Internal Server Error by the handler */\n type: 'InternalError';\n }\n | {\n /** Will be translated to 404 Not Found by the handler */\n type: 'NotFound';\n }\n | {\n /** Will be translated to 416 Range Not Satisfiable by the handler */\n type: 'RangeNotSatisfiable';\n /** Total file size in bytes */\n size: number;\n }\n | {\n /** Will be translated to 200 OK or 206 Partial Content by the handler */\n type: 'Ok';\n /** Total file size in bytes */\n size: number;\n /** File range translated from HTTP range */\n range: FileRange;\n /** Stream of file content, undefined for HEAD requests */\n data?: Readable;\n }\n\n/** Common options for object store creation */\nexport interface ObjectStoreOptions {\n /** Logger instance, no logging is performed when not provided */\n logger?: Logger;\n}\n\n/** Options for file system object store creation */\nexport interface FsStoreOptions extends ObjectStoreOptions {\n /** Local directory to serve files from */\n rootDir: string;\n}\n\n/** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */\nexport abstract class ObjectStore {\n protected readonly logger: Logger;\n\n constructor(options: ObjectStoreOptions) {\n this.logger = options.logger ?? (() => {});\n }\n\n /** Translate HTTP range to file range, @returns null if the range is not satisfiable */\n protected translate(fileSize: number, range?: HttpRange): FileRange | null {\n if (!range) return { start: 0, end: fileSize - 1 };\n switch (range.type) {\n case 'bounded':\n if (range.end >= fileSize) return null;\n return { start: range.start, end: range.end };\n case 'offset':\n if (range.offset >= fileSize) return null;\n return { start: range.offset, end: fileSize - 1 };\n case 'suffix':\n if (range.suffix > fileSize) return null;\n return { start: fileSize - range.suffix, end: fileSize - 1 };\n }\n }\n\n /**\n * Proxy HTTP(S) request for parquet file to object store.\n * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler\n * Callback API is used so that ObjectStore can limit the number of concurrent requests.\n */\n abstract request(\n filename: ParquetFileName,\n params: {\n method: HttpMethod;\n range?: HttpRange;\n signal: AbortSignal;\n callback: (response: ObjectStoreResponse) => Promise<void>;\n }\n ): void;\n}\n\n/** Object store base URL in format accepted by Apache DataFusion and DuckDB */\nexport type ObjectStoreUrl = Branded<string, 'PFrameInternal.ObjectStoreUrl'>;\n\n/** HTTP(S) request handler creation options */\nexport type RequestHandlerOptions = {\n /** Object store to serve files from, @see HttpHelpers.createFsStore */\n store: ObjectStore;\n /** Here will go caching options... */\n}\n\n/** Server configuration options */\nexport type HttpServerOptions = {\n /** HTTP(S) request handler function, @see HttpHelpers.createRequestHandler */\n handler: RequestListener;\n /** Port to bind to, @default 0 for auto-assignment */\n port?: number;\n /** Do not apply authorization middleware to @param handler */\n noAuth?: true;\n /** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */\n noHttps?: true;\n};\n\n/**\n * Long unique opaque string for use in Bearer authorization header\n * \n * @example\n * ```ts\n * request.setHeader('Authorization', `Bearer ${authToken}`);\n * ```\n */\nexport type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthorizationToken'>;\n\n/**\n * TLS certificate in PEM format\n * \n * @example\n * ```txt\n * -----BEGIN CERTIFICATE-----\n * MIIC2zCCAcOgAwIBAgIJaVW7...\n * ...\n * ...Yf9CRK8fgnukKM7TJ\n * -----END CERTIFICATE-----\n * ```\n */\nexport type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;\n\n/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */\nexport type ParquetServerInfo = {\n /** URL of the parquet HTTP(S) server formatted as `http{s}://<host>:<port>/` */\n url: ObjectStoreUrl;\n /** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */\n authToken?: HttpAuthorizationToken;\n /** Encoded CA certificate of HTTPS server, undefined when @see HttpServerOptions.noHttps flag is set */\n encodedCaCert?: Base64Encoded<PemCertificate>;\n};\n\n/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */\nexport interface HttpServer {\n /** Server configuration information for initiating connections from clients */\n get info(): ParquetServerInfo;\n /** Promise that resolves when the server is stopped */\n get stopped(): Promise<void>;\n /** Request server stop, returns the same promise as @see HttpServer.stopped */\n stop(): Promise<void>;\n}\n\n/** List of HTTP(S) related helper functions exposed by PFrame module */\nexport interface HttpHelpers {\n /**\n * Create an object store for serving files from a local directory.\n * Rejects if the provided path does not exist or is not a directory.\n */\n createFsStore(options: FsStoreOptions): Promise<ObjectStore>;\n\n /**\n * Create an HTTP request handler for serving files from an object store.\n * Accepts only paths of the form `/<filename>.parquet`, returns 410 otherwise.\n * Assumes that files are immutable (and sets cache headers accordingly).\n */\n createRequestHandler(options: RequestHandlerOptions): RequestListener;\n\n /**\n * Serve HTTP(S) requests using the provided handler on localhost port.\n * @returns promise that resolves when the server has stopped.\n *\n * @example\n * ```ts\n * const rootDir = '/path/to/directory/with/parquet/files';\n *\n * let store = await HttpHelpers.createFsStore({ rootDir }).catch((err: unknown) => {\n * throw new Error(`Failed to create file store for ${rootDir} - ${ensureError(err)}`);\n * });\n *\n * const server = await HttpHelpers.createHttpServer({\n * handler: HttpHelpers.createRequestHandler({ store }),\n * }).catch((err: unknown) => {\n * throw new Error(`Failed to start HTTPS server - ${ensureError(err)}`);\n * });\n *\n * const { url, authToken, encodedCaCert } = server.info;\n *\n * await server.stop();\n * ```\n */\n createHttpServer(options: HttpServerOptions): Promise<HttpServer>;\n}\n"],"names":[],"mappings":";;AAwGA;MACsB,WAAW,CAAA;AACZ,IAAA,MAAM;AAEzB,IAAA,WAAA,CAAY,OAA2B,EAAA;AACrC,QAAA,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,MAAK,EAAE,CAAC,CAAC;IAC5C;;IAGU,SAAS,CAAC,QAAgB,EAAE,KAAiB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AAClD,QAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,YAAA,KAAK,SAAS;AACZ,gBAAA,IAAI,KAAK,CAAC,GAAG,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACtC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE;AAC/C,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACzC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AACnD,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACxC,gBAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;;IAElE;AAgBD;;;;"}
|
|
@@ -121,7 +121,7 @@ export type HttpServerOptions = {
|
|
|
121
121
|
/** Do not apply authorization middleware to @param handler */
|
|
122
122
|
noAuth?: true;
|
|
123
123
|
/** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */
|
|
124
|
-
|
|
124
|
+
noHttps?: true;
|
|
125
125
|
};
|
|
126
126
|
/**
|
|
127
127
|
* Long unique opaque string for use in Bearer authorization header
|
|
@@ -145,14 +145,19 @@ export type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthori
|
|
|
145
145
|
* ```
|
|
146
146
|
*/
|
|
147
147
|
export type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;
|
|
148
|
+
/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
|
|
149
|
+
export type ParquetServerInfo = {
|
|
150
|
+
/** URL of the parquet HTTP(S) server formatted as `http{s}://<host>:<port>/` */
|
|
151
|
+
url: ObjectStoreUrl;
|
|
152
|
+
/** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */
|
|
153
|
+
authToken?: HttpAuthorizationToken;
|
|
154
|
+
/** Encoded CA certificate of HTTPS server, undefined when @see HttpServerOptions.noHttps flag is set */
|
|
155
|
+
encodedCaCert?: Base64Encoded<PemCertificate>;
|
|
156
|
+
};
|
|
148
157
|
/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */
|
|
149
158
|
export interface HttpServer {
|
|
150
|
-
/** Server
|
|
151
|
-
get
|
|
152
|
-
/** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */
|
|
153
|
-
get authToken(): HttpAuthorizationToken | undefined;
|
|
154
|
-
/** Base64-encoded CA certificate in PEM format, undefined when @see HttpServerOptions.http flag is set */
|
|
155
|
-
get encodedCaCert(): Base64Encoded<PemCertificate> | undefined;
|
|
159
|
+
/** Server configuration information for initiating connections from clients */
|
|
160
|
+
get info(): ParquetServerInfo;
|
|
156
161
|
/** Promise that resolves when the server is stopped */
|
|
157
162
|
get stopped(): Promise<void>;
|
|
158
163
|
/** Request server stop, returns the same promise as @see HttpServer.stopped */
|
|
@@ -184,12 +189,12 @@ export interface HttpHelpers {
|
|
|
184
189
|
* });
|
|
185
190
|
*
|
|
186
191
|
* const server = await HttpHelpers.createHttpServer({
|
|
187
|
-
* handler: HttpHelpers.createRequestHandler(store),
|
|
192
|
+
* handler: HttpHelpers.createRequestHandler({ store }),
|
|
188
193
|
* }).catch((err: unknown) => {
|
|
189
|
-
* throw new Error(`Failed to start
|
|
194
|
+
* throw new Error(`Failed to start HTTPS server - ${ensureError(err)}`);
|
|
190
195
|
* });
|
|
191
196
|
*
|
|
192
|
-
* const {
|
|
197
|
+
* const { url, authToken, encodedCaCert } = server.info;
|
|
193
198
|
*
|
|
194
199
|
* await server.stop();
|
|
195
200
|
* ```
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http_helpers.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAC9E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,wBAAwB;AACxB,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE,gCAAgC,CAAC,CAAC;AAE7F,MAAM,MAAM,SAAS,GAAG;IACtB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,CAAA;AAED,2FAA2F;AAC3F,MAAM,MAAM,SAAS,GACjB;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,SAAS,CAAC;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,yCAAyC;AACzC,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,CAAC;AAExC,sCAAsC;AACtC,MAAM,MAAM,mBAAmB,GAC3B;IACE,qEAAqE;IACrE,IAAI,EAAE,eAAe,CAAC;CACvB,GACD;IACE,yDAAyD;IACzD,IAAI,EAAE,UAAU,CAAC;CAClB,GACD;IACE,qEAAqE;IACrE,IAAI,EAAE,qBAAqB,CAAC;IAC5B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,yEAAyE;IACzE,IAAI,EAAE,IAAI,CAAC;IACX,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,KAAK,EAAE,SAAS,CAAC;IACjB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,QAAQ,CAAC;CACjB,CAAA;AAEL,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,oDAAoD;AACpD,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iGAAiG;AACjG,8BAAsB,WAAW;IAC/B,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEtB,OAAO,EAAE,kBAAkB;IAIvC,wFAAwF;IACxF,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,IAAI;IAe1E;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CACd,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE;QACN,MAAM,EAAE,UAAU,CAAC;QACnB,KAAK,CAAC,EAAE,SAAS,CAAC;QAClB,MAAM,EAAE,WAAW,CAAC;QACpB,QAAQ,EAAE,CAAC,QAAQ,EAAE,mBAAmB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;KAC5D,GACA,IAAI;CACR;AAED,+EAA+E;AAC/E,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,+CAA+C;AAC/C,MAAM,MAAM,qBAAqB,GAAG;IAClC,uEAAuE;IACvE,KAAK,EAAE,WAAW,CAAC;CAEpB,CAAA;AAED,mCAAmC;AACnC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,8EAA8E;IAC9E,OAAO,EAAE,eAAe,CAAC;IACzB,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,kFAAkF;IAClF,
|
|
1
|
+
{"version":3,"file":"http_helpers.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAC9E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,wBAAwB;AACxB,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE,gCAAgC,CAAC,CAAC;AAE7F,MAAM,MAAM,SAAS,GAAG;IACtB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,CAAA;AAED,2FAA2F;AAC3F,MAAM,MAAM,SAAS,GACjB;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,SAAS,CAAC;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;CACb,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;OAQG;IACH,IAAI,EAAE,QAAQ,CAAC;IACf,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,yCAAyC;AACzC,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,CAAC;AAExC,sCAAsC;AACtC,MAAM,MAAM,mBAAmB,GAC3B;IACE,qEAAqE;IACrE,IAAI,EAAE,eAAe,CAAC;CACvB,GACD;IACE,yDAAyD;IACzD,IAAI,EAAE,UAAU,CAAC;CAClB,GACD;IACE,qEAAqE;IACrE,IAAI,EAAE,qBAAqB,CAAC;IAC5B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,yEAAyE;IACzE,IAAI,EAAE,IAAI,CAAC;IACX,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,KAAK,EAAE,SAAS,CAAC;IACjB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,QAAQ,CAAC;CACjB,CAAA;AAEL,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,oDAAoD;AACpD,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iGAAiG;AACjG,8BAAsB,WAAW;IAC/B,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEtB,OAAO,EAAE,kBAAkB;IAIvC,wFAAwF;IACxF,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,IAAI;IAe1E;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CACd,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE;QACN,MAAM,EAAE,UAAU,CAAC;QACnB,KAAK,CAAC,EAAE,SAAS,CAAC;QAClB,MAAM,EAAE,WAAW,CAAC;QACpB,QAAQ,EAAE,CAAC,QAAQ,EAAE,mBAAmB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;KAC5D,GACA,IAAI;CACR;AAED,+EAA+E;AAC/E,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,+CAA+C;AAC/C,MAAM,MAAM,qBAAqB,GAAG;IAClC,uEAAuE;IACvE,KAAK,EAAE,WAAW,CAAC;CAEpB,CAAA;AAED,mCAAmC;AACnC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,8EAA8E;IAC9E,OAAO,EAAE,eAAe,CAAC;IACzB,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,kFAAkF;IAClF,OAAO,CAAC,EAAE,IAAI,CAAC;CAChB,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,EAAE,uCAAuC,CAAC,CAAC;AAE9F;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;AAE9E,uFAAuF;AACvF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,gFAAgF;IAChF,GAAG,EAAE,cAAc,CAAC;IACpB,sGAAsG;IACtG,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,wGAAwG;IACxG,aAAa,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CAC/C,CAAC;AAEF,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,+EAA+E;IAC/E,IAAI,IAAI,IAAI,iBAAiB,CAAC;IAC9B,uDAAuD;IACvD,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,+EAA+E;IAC/E,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE7D;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,eAAe,CAAC;IAEtE;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACnE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http_helpers.js","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n /** Will be translated to 500 Internal Server Error by the handler */\n type: 'InternalError';\n }\n | {\n /** Will be translated to 404 Not Found by the handler */\n type: 'NotFound';\n }\n | {\n /** Will be translated to 416 Range Not Satisfiable by the handler */\n type: 'RangeNotSatisfiable';\n /** Total file size in bytes */\n size: number;\n }\n | {\n /** Will be translated to 200 OK or 206 Partial Content by the handler */\n type: 'Ok';\n /** Total file size in bytes */\n size: number;\n /** File range translated from HTTP range */\n range: FileRange;\n /** Stream of file content, undefined for HEAD requests */\n data?: Readable;\n }\n\n/** Common options for object store creation */\nexport interface ObjectStoreOptions {\n /** Logger instance, no logging is performed when not provided */\n logger?: Logger;\n}\n\n/** Options for file system object store creation */\nexport interface FsStoreOptions extends ObjectStoreOptions {\n /** Local directory to serve files from */\n rootDir: string;\n}\n\n/** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */\nexport abstract class ObjectStore {\n protected readonly logger: Logger;\n\n constructor(options: ObjectStoreOptions) {\n this.logger = options.logger ?? (() => {});\n }\n\n /** Translate HTTP range to file range, @returns null if the range is not satisfiable */\n protected translate(fileSize: number, range?: HttpRange): FileRange | null {\n if (!range) return { start: 0, end: fileSize - 1 };\n switch (range.type) {\n case 'bounded':\n if (range.end >= fileSize) return null;\n return { start: range.start, end: range.end };\n case 'offset':\n if (range.offset >= fileSize) return null;\n return { start: range.offset, end: fileSize - 1 };\n case 'suffix':\n if (range.suffix > fileSize) return null;\n return { start: fileSize - range.suffix, end: fileSize - 1 };\n }\n }\n\n /**\n * Proxy HTTP(S) request for parquet file to object store.\n * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler\n * Callback API is used so that ObjectStore can limit the number of concurrent requests.\n */\n abstract request(\n filename: ParquetFileName,\n params: {\n method: HttpMethod;\n range?: HttpRange;\n signal: AbortSignal;\n callback: (response: ObjectStoreResponse) => Promise<void>;\n }\n ): void;\n}\n\n/** Object store base URL in format accepted by Apache DataFusion and DuckDB */\nexport type ObjectStoreUrl = Branded<string, 'PFrameInternal.ObjectStoreUrl'>;\n\n/** HTTP(S) request handler creation options */\nexport type RequestHandlerOptions = {\n /** Object store to serve files from, @see HttpHelpers.createFsStore */\n store: ObjectStore;\n /** Here will go caching options... */\n}\n\n/** Server configuration options */\nexport type HttpServerOptions = {\n /** HTTP(S) request handler function, @see HttpHelpers.createRequestHandler */\n handler: RequestListener;\n /** Port to bind to, @default 0 for auto-assignment */\n port?: number;\n /** Do not apply authorization middleware to @param handler */\n noAuth?: true;\n /** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */\n
|
|
1
|
+
{"version":3,"file":"http_helpers.js","sources":["../../../src/pframe/internal_api/http_helpers.ts"],"sourcesContent":["import type { Readable } from 'node:stream';\nimport type { RequestListener } from 'node:http';\nimport type { Branded, Base64Encoded } from '@milaboratories/pl-model-common';\nimport type { Logger } from './common';\n\n/** Parquet file name */\nexport type ParquetFileName = Branded<`${string}.parquet`, 'PFrameInternal.ParquetFileName'>;\n\nexport type FileRange = {\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n}\n\n/** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */\nexport type HttpRange =\n | {\n /**\n * Get file content in the specified byte range\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=0-1023\n * ```\n */\n type: 'bounded';\n /** Start byte position (inclusive) */\n start: number;\n /** End byte position (inclusive) */\n end: number;\n }\n | {\n /**\n * Get byte range starting from the specified offset\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=1024-\n * ```\n */\n type: 'offset';\n /** Start byte position (inclusive) */\n offset: number;\n }\n | {\n /**\n * Get byte range starting from the specified suffix\n * \n * @example\n * ```\n * GET /file.parquet HTTP/1.1\n * Range: bytes=-1024\n * ```\n */\n type: 'suffix';\n /** End byte position (inclusive) */\n suffix: number;\n };\n\n/** HTTP method passed to object store */\nexport type HttpMethod = 'GET' | 'HEAD';\n\n/** HTTP response from object store */\nexport type ObjectStoreResponse =\n | {\n /** Will be translated to 500 Internal Server Error by the handler */\n type: 'InternalError';\n }\n | {\n /** Will be translated to 404 Not Found by the handler */\n type: 'NotFound';\n }\n | {\n /** Will be translated to 416 Range Not Satisfiable by the handler */\n type: 'RangeNotSatisfiable';\n /** Total file size in bytes */\n size: number;\n }\n | {\n /** Will be translated to 200 OK or 206 Partial Content by the handler */\n type: 'Ok';\n /** Total file size in bytes */\n size: number;\n /** File range translated from HTTP range */\n range: FileRange;\n /** Stream of file content, undefined for HEAD requests */\n data?: Readable;\n }\n\n/** Common options for object store creation */\nexport interface ObjectStoreOptions {\n /** Logger instance, no logging is performed when not provided */\n logger?: Logger;\n}\n\n/** Options for file system object store creation */\nexport interface FsStoreOptions extends ObjectStoreOptions {\n /** Local directory to serve files from */\n rootDir: string;\n}\n\n/** File system abstraction for request handler factory, @see HttpHelpers.createRequestHandler */\nexport abstract class ObjectStore {\n protected readonly logger: Logger;\n\n constructor(options: ObjectStoreOptions) {\n this.logger = options.logger ?? (() => {});\n }\n\n /** Translate HTTP range to file range, @returns null if the range is not satisfiable */\n protected translate(fileSize: number, range?: HttpRange): FileRange | null {\n if (!range) return { start: 0, end: fileSize - 1 };\n switch (range.type) {\n case 'bounded':\n if (range.end >= fileSize) return null;\n return { start: range.start, end: range.end };\n case 'offset':\n if (range.offset >= fileSize) return null;\n return { start: range.offset, end: fileSize - 1 };\n case 'suffix':\n if (range.suffix > fileSize) return null;\n return { start: fileSize - range.suffix, end: fileSize - 1 };\n }\n }\n\n /**\n * Proxy HTTP(S) request for parquet file to object store.\n * Callback promise resolves when stream is closed by handler @see HttpHelpers.createRequestHandler\n * Callback API is used so that ObjectStore can limit the number of concurrent requests.\n */\n abstract request(\n filename: ParquetFileName,\n params: {\n method: HttpMethod;\n range?: HttpRange;\n signal: AbortSignal;\n callback: (response: ObjectStoreResponse) => Promise<void>;\n }\n ): void;\n}\n\n/** Object store base URL in format accepted by Apache DataFusion and DuckDB */\nexport type ObjectStoreUrl = Branded<string, 'PFrameInternal.ObjectStoreUrl'>;\n\n/** HTTP(S) request handler creation options */\nexport type RequestHandlerOptions = {\n /** Object store to serve files from, @see HttpHelpers.createFsStore */\n store: ObjectStore;\n /** Here will go caching options... */\n}\n\n/** Server configuration options */\nexport type HttpServerOptions = {\n /** HTTP(S) request handler function, @see HttpHelpers.createRequestHandler */\n handler: RequestListener;\n /** Port to bind to, @default 0 for auto-assignment */\n port?: number;\n /** Do not apply authorization middleware to @param handler */\n noAuth?: true;\n /** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */\n noHttps?: true;\n};\n\n/**\n * Long unique opaque string for use in Bearer authorization header\n * \n * @example\n * ```ts\n * request.setHeader('Authorization', `Bearer ${authToken}`);\n * ```\n */\nexport type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthorizationToken'>;\n\n/**\n * TLS certificate in PEM format\n * \n * @example\n * ```txt\n * -----BEGIN CERTIFICATE-----\n * MIIC2zCCAcOgAwIBAgIJaVW7...\n * ...\n * ...Yf9CRK8fgnukKM7TJ\n * -----END CERTIFICATE-----\n * ```\n */\nexport type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;\n\n/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */\nexport type ParquetServerInfo = {\n /** URL of the parquet HTTP(S) server formatted as `http{s}://<host>:<port>/` */\n url: ObjectStoreUrl;\n /** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */\n authToken?: HttpAuthorizationToken;\n /** Encoded CA certificate of HTTPS server, undefined when @see HttpServerOptions.noHttps flag is set */\n encodedCaCert?: Base64Encoded<PemCertificate>;\n};\n\n/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */\nexport interface HttpServer {\n /** Server configuration information for initiating connections from clients */\n get info(): ParquetServerInfo;\n /** Promise that resolves when the server is stopped */\n get stopped(): Promise<void>;\n /** Request server stop, returns the same promise as @see HttpServer.stopped */\n stop(): Promise<void>;\n}\n\n/** List of HTTP(S) related helper functions exposed by PFrame module */\nexport interface HttpHelpers {\n /**\n * Create an object store for serving files from a local directory.\n * Rejects if the provided path does not exist or is not a directory.\n */\n createFsStore(options: FsStoreOptions): Promise<ObjectStore>;\n\n /**\n * Create an HTTP request handler for serving files from an object store.\n * Accepts only paths of the form `/<filename>.parquet`, returns 410 otherwise.\n * Assumes that files are immutable (and sets cache headers accordingly).\n */\n createRequestHandler(options: RequestHandlerOptions): RequestListener;\n\n /**\n * Serve HTTP(S) requests using the provided handler on localhost port.\n * @returns promise that resolves when the server has stopped.\n *\n * @example\n * ```ts\n * const rootDir = '/path/to/directory/with/parquet/files';\n *\n * let store = await HttpHelpers.createFsStore({ rootDir }).catch((err: unknown) => {\n * throw new Error(`Failed to create file store for ${rootDir} - ${ensureError(err)}`);\n * });\n *\n * const server = await HttpHelpers.createHttpServer({\n * handler: HttpHelpers.createRequestHandler({ store }),\n * }).catch((err: unknown) => {\n * throw new Error(`Failed to start HTTPS server - ${ensureError(err)}`);\n * });\n *\n * const { url, authToken, encodedCaCert } = server.info;\n *\n * await server.stop();\n * ```\n */\n createHttpServer(options: HttpServerOptions): Promise<HttpServer>;\n}\n"],"names":[],"mappings":"AAwGA;MACsB,WAAW,CAAA;AACZ,IAAA,MAAM;AAEzB,IAAA,WAAA,CAAY,OAA2B,EAAA;AACrC,QAAA,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,MAAK,EAAE,CAAC,CAAC;IAC5C;;IAGU,SAAS,CAAC,QAAgB,EAAE,KAAiB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AAClD,QAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,YAAA,KAAK,SAAS;AACZ,gBAAA,IAAI,KAAK,CAAC,GAAG,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACtC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE;AAC/C,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACzC,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;AACnD,YAAA,KAAK,QAAQ;AACX,gBAAA,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;AAAE,oBAAA,OAAO,IAAI;AACxC,gBAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,CAAC,EAAE;;IAElE;AAgBD;;;;"}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { PFrameFactoryAPIV4 } from './api_factory';
|
|
2
2
|
import type { PFrameReadAPIV9 } from './api_read';
|
|
3
3
|
import type { Logger } from './common';
|
|
4
|
-
export interface PFrameV10 extends PFrameFactoryAPIV3, PFrameReadAPIV9 {
|
|
5
|
-
}
|
|
6
4
|
export interface PFrameV11 extends PFrameFactoryAPIV4, PFrameReadAPIV9 {
|
|
7
5
|
}
|
|
8
6
|
export type PFrameOptions = {
|
|
@@ -12,22 +10,6 @@ export type PFrameOptions = {
|
|
|
12
10
|
logger?: Logger;
|
|
13
11
|
};
|
|
14
12
|
/** List of PFrame management functions exposed by PFrame module */
|
|
15
|
-
export interface PFrameFactory {
|
|
16
|
-
/**
|
|
17
|
-
* Create a new PFrame instance.
|
|
18
|
-
* @warning Use concurrency limiting to avoid OOM crashes when multiple instances are simultaneously in use.
|
|
19
|
-
*/
|
|
20
|
-
createPFrame(options: PFrameOptions): PFrameV10;
|
|
21
|
-
/**
|
|
22
|
-
* Dump active allocations from all PFrames instances in pprof format.
|
|
23
|
-
* The result of this function should be saved as `profile.pb.gz`.
|
|
24
|
-
* Use {@link https://pprof.me/} or {@link https://www.speedscope.app/}
|
|
25
|
-
* to view the allocation flamechart.
|
|
26
|
-
* @warning This method will always reject on Windows!
|
|
27
|
-
*/
|
|
28
|
-
pprofDump: () => Promise<Uint8Array>;
|
|
29
|
-
}
|
|
30
|
-
/** List of PFrame management functions exposed by PFrame module */
|
|
31
13
|
export interface PFrameFactoryV2 {
|
|
32
14
|
/**
|
|
33
15
|
* Create a new PFrame instance.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pframe.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/pframe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,
|
|
1
|
+
{"version":3,"file":"pframe.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/pframe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,MAAM,WAAW,SAAU,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAEzE,MAAM,MAAM,aAAa,GAAG;IAC1B,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAA;AAED,mEAAmE;AACnE,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,CAAC;IAEhD;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;CACtC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-model-middle-layer",
|
|
3
|
-
"version": "1.8.
|
|
3
|
+
"version": "1.8.19",
|
|
4
4
|
"description": "Common model between middle layer and non-block UI code",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -24,9 +24,9 @@
|
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"typescript": "~5.6.3",
|
|
27
|
-
"@milaboratories/build-configs": "1.0.8",
|
|
28
27
|
"@milaboratories/ts-builder": "1.0.5",
|
|
29
|
-
"@milaboratories/ts-configs": "1.0.6"
|
|
28
|
+
"@milaboratories/ts-configs": "1.0.6",
|
|
29
|
+
"@milaboratories/build-configs": "1.0.8"
|
|
30
30
|
},
|
|
31
31
|
"scripts": {
|
|
32
32
|
"type-check": "ts-builder types --target node",
|
|
@@ -10,9 +10,7 @@ import {
|
|
|
10
10
|
} from '@milaboratories/pl-model-common';
|
|
11
11
|
import {
|
|
12
12
|
HttpHelpers,
|
|
13
|
-
|
|
14
|
-
ObjectStoreUrl,
|
|
15
|
-
PemCertificate,
|
|
13
|
+
ParquetServerInfo,
|
|
16
14
|
} from './http_helpers';
|
|
17
15
|
|
|
18
16
|
/** Abstract identifier of a data blob that can be requested from the storage backend */
|
|
@@ -23,31 +21,6 @@ export type PFrameBlobId = string;
|
|
|
23
21
|
* {@link BlobPathResolver} as soon as blob materialized in the file system. */
|
|
24
22
|
export type FilePath = string;
|
|
25
23
|
|
|
26
|
-
/** Data source allows PFrame to retrieve the data blobs for columns with assigned data info. */
|
|
27
|
-
export type PFrameDataSource = {
|
|
28
|
-
/**
|
|
29
|
-
* PFrame may notify storage backend about its plans to use particular blobs in the future.
|
|
30
|
-
* Storage backend will do its best to preload specified blob so the subsequent
|
|
31
|
-
* {@link resolveBlob} will quickly return preloaded file path.
|
|
32
|
-
*/
|
|
33
|
-
preloadBlob(blobIds: PFrameBlobId[]): Promise<void>;
|
|
34
|
-
|
|
35
|
-
/** Returns raw blob data given the blob id from {@link DataInfo}. */
|
|
36
|
-
resolveBlobContent(blobId: PFrameBlobId): Promise<Uint8Array>;
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
|
|
40
|
-
export type ParquetServerConfig = {
|
|
41
|
-
/** URL of the parquet HTTP(S) server */
|
|
42
|
-
url: ObjectStoreUrl;
|
|
43
|
-
|
|
44
|
-
/** Authorization token for Bearer scheme */
|
|
45
|
-
authToken?: HttpAuthorizationToken;
|
|
46
|
-
|
|
47
|
-
/** CA certificate of HTTPS server */
|
|
48
|
-
caCert?: Base64Encoded<PemCertificate>;
|
|
49
|
-
};
|
|
50
|
-
|
|
51
24
|
/** Data source allows PFrame to retrieve the data blobs for columns with assigned data info. */
|
|
52
25
|
export interface PFrameDataSourceV2 {
|
|
53
26
|
/**
|
|
@@ -64,7 +37,7 @@ export interface PFrameDataSourceV2 {
|
|
|
64
37
|
* Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer}
|
|
65
38
|
* When not provided, parquet BlobIds would be treated as local file paths.
|
|
66
39
|
*/
|
|
67
|
-
parquetServer?:
|
|
40
|
+
parquetServer?: ParquetServerInfo;
|
|
68
41
|
}
|
|
69
42
|
|
|
70
43
|
/**
|
|
@@ -79,32 +52,6 @@ export type DataInfo<Blob> =
|
|
|
79
52
|
| BinaryPartitionedDataInfo<Blob>
|
|
80
53
|
| ParquetPartitionedDataInfo<ParquetChunk<Blob>>;
|
|
81
54
|
|
|
82
|
-
/** API exposed by PFrames library allowing to create and provide data for
|
|
83
|
-
* PFrame objects */
|
|
84
|
-
export interface PFrameFactoryAPIV3 extends Disposable {
|
|
85
|
-
/** Associates data source with this PFrame */
|
|
86
|
-
setDataSource(dataSource: PFrameDataSource): void;
|
|
87
|
-
|
|
88
|
-
/** Adds PColumn without data info */
|
|
89
|
-
addColumnSpec(columnId: PObjectId, columnSpec: PColumnSpec): void;
|
|
90
|
-
|
|
91
|
-
/**
|
|
92
|
-
* Assign data info to the specified PColumn.
|
|
93
|
-
* For parquet data info, schema resolution via network is performed during this call.
|
|
94
|
-
*/
|
|
95
|
-
setColumnData(
|
|
96
|
-
columnId: PObjectId,
|
|
97
|
-
dataInfo: DataInfo<PFrameBlobId>,
|
|
98
|
-
options?: {
|
|
99
|
-
signal?: AbortSignal,
|
|
100
|
-
}
|
|
101
|
-
): Promise<void>;
|
|
102
|
-
|
|
103
|
-
/** Releases all the data previously added to PFrame using methods above,
|
|
104
|
-
* any interactions with disposed PFrame will result in exception */
|
|
105
|
-
dispose(): void;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
55
|
/** API exposed by PFrames library allowing to create and provide data for
|
|
109
56
|
* PFrame objects */
|
|
110
57
|
export interface PFrameFactoryAPIV4 extends Disposable {
|
|
@@ -161,7 +161,7 @@ export type HttpServerOptions = {
|
|
|
161
161
|
/** Do not apply authorization middleware to @param handler */
|
|
162
162
|
noAuth?: true;
|
|
163
163
|
/** Downgrade default HTTPS server to plain HTTP, @warning use only for testing */
|
|
164
|
-
|
|
164
|
+
noHttps?: true;
|
|
165
165
|
};
|
|
166
166
|
|
|
167
167
|
/**
|
|
@@ -188,14 +188,20 @@ export type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthori
|
|
|
188
188
|
*/
|
|
189
189
|
export type PemCertificate = Branded<string, 'PFrameInternal.PemCertificate'>;
|
|
190
190
|
|
|
191
|
+
/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
|
|
192
|
+
export type ParquetServerInfo = {
|
|
193
|
+
/** URL of the parquet HTTP(S) server formatted as `http{s}://<host>:<port>/` */
|
|
194
|
+
url: ObjectStoreUrl;
|
|
195
|
+
/** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */
|
|
196
|
+
authToken?: HttpAuthorizationToken;
|
|
197
|
+
/** Encoded CA certificate of HTTPS server, undefined when @see HttpServerOptions.noHttps flag is set */
|
|
198
|
+
encodedCaCert?: Base64Encoded<PemCertificate>;
|
|
199
|
+
};
|
|
200
|
+
|
|
191
201
|
/** HTTP(S) server information and controls, @see HttpHelpers.createHttpServer */
|
|
192
202
|
export interface HttpServer {
|
|
193
|
-
/** Server
|
|
194
|
-
get
|
|
195
|
-
/** Authorization token for Bearer scheme, undefined when @see HttpServerOptions.noAuth flag is set */
|
|
196
|
-
get authToken(): HttpAuthorizationToken | undefined;
|
|
197
|
-
/** Base64-encoded CA certificate in PEM format, undefined when @see HttpServerOptions.http flag is set */
|
|
198
|
-
get encodedCaCert(): Base64Encoded<PemCertificate> | undefined;
|
|
203
|
+
/** Server configuration information for initiating connections from clients */
|
|
204
|
+
get info(): ParquetServerInfo;
|
|
199
205
|
/** Promise that resolves when the server is stopped */
|
|
200
206
|
get stopped(): Promise<void>;
|
|
201
207
|
/** Request server stop, returns the same promise as @see HttpServer.stopped */
|
|
@@ -230,12 +236,12 @@ export interface HttpHelpers {
|
|
|
230
236
|
* });
|
|
231
237
|
*
|
|
232
238
|
* const server = await HttpHelpers.createHttpServer({
|
|
233
|
-
* handler: HttpHelpers.createRequestHandler(store),
|
|
239
|
+
* handler: HttpHelpers.createRequestHandler({ store }),
|
|
234
240
|
* }).catch((err: unknown) => {
|
|
235
|
-
* throw new Error(`Failed to start
|
|
241
|
+
* throw new Error(`Failed to start HTTPS server - ${ensureError(err)}`);
|
|
236
242
|
* });
|
|
237
243
|
*
|
|
238
|
-
* const {
|
|
244
|
+
* const { url, authToken, encodedCaCert } = server.info;
|
|
239
245
|
*
|
|
240
246
|
* await server.stop();
|
|
241
247
|
* ```
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { PFrameFactoryAPIV4 } from './api_factory';
|
|
2
2
|
import type { PFrameReadAPIV9 } from './api_read';
|
|
3
3
|
import type { Logger } from './common';
|
|
4
4
|
|
|
5
|
-
export interface PFrameV10 extends PFrameFactoryAPIV3, PFrameReadAPIV9 {}
|
|
6
|
-
|
|
7
5
|
export interface PFrameV11 extends PFrameFactoryAPIV4, PFrameReadAPIV9 {}
|
|
8
6
|
|
|
9
7
|
export type PFrameOptions = {
|
|
@@ -13,24 +11,6 @@ export type PFrameOptions = {
|
|
|
13
11
|
logger?: Logger;
|
|
14
12
|
}
|
|
15
13
|
|
|
16
|
-
/** List of PFrame management functions exposed by PFrame module */
|
|
17
|
-
export interface PFrameFactory {
|
|
18
|
-
/**
|
|
19
|
-
* Create a new PFrame instance.
|
|
20
|
-
* @warning Use concurrency limiting to avoid OOM crashes when multiple instances are simultaneously in use.
|
|
21
|
-
*/
|
|
22
|
-
createPFrame(options: PFrameOptions): PFrameV10;
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Dump active allocations from all PFrames instances in pprof format.
|
|
26
|
-
* The result of this function should be saved as `profile.pb.gz`.
|
|
27
|
-
* Use {@link https://pprof.me/} or {@link https://www.speedscope.app/}
|
|
28
|
-
* to view the allocation flamechart.
|
|
29
|
-
* @warning This method will always reject on Windows!
|
|
30
|
-
*/
|
|
31
|
-
pprofDump: () => Promise<Uint8Array>;
|
|
32
|
-
};
|
|
33
|
-
|
|
34
14
|
/** List of PFrame management functions exposed by PFrame module */
|
|
35
15
|
export interface PFrameFactoryV2 {
|
|
36
16
|
/**
|