@dpuse/dpuse-shared 1.0.210 → 1.0.212

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
  * Defines the configuration metadata for a connector. Used for validation
3
3
  * of connector manifests and capability discovery at runtime.
4
4
  */
5
- export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
5
+ export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
6
6
  export declare const connectorCategoryConfigSchema: import('valibot').StrictObjectSchema<{
7
7
  readonly label: import('valibot').StrictObjectSchema<{
8
8
  readonly en: import('valibot').StringSchema<undefined>;
@@ -25,7 +25,7 @@ export declare const connectorUsageConfigSchema: import('valibot').StrictObjectS
25
25
  }, undefined>;
26
26
  export declare const connectorConfigSchema: import('valibot').StrictObjectSchema<{
27
27
  readonly typeId: import('valibot').LiteralSchema<"connector", undefined>;
28
- readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
28
+ readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
29
29
  readonly category: import('valibot').NullableSchema<import('valibot').StrictObjectSchema<{
30
30
  readonly label: import('valibot').StrictObjectSchema<{
31
31
  readonly en: import('valibot').StringSchema<undefined>;
@@ -8,7 +8,6 @@ import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, Ob
8
8
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
9
9
  export { connectorConfigSchema } from './connectorConfig.schema';
10
10
  export interface ConnectorInterface {
11
- abortOperation?(): void;
12
11
  auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
13
12
  closeStore?(options: CloseStoreOptions): Promise<void>;
14
13
  createObject?(options: CreateObjectOptions): Promise<void>;
@@ -24,9 +23,7 @@ export interface ConnectorInterface {
24
23
  previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
25
24
  queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
26
25
  removeRecords?(options: RemoveRecordsOptions): Promise<void>;
27
- retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
28
- retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsedRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
29
- complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
26
+ retrieveRecords?(options: RetrieveRecordsOptions): AsyncGenerator<RetrieveRecordsChunk, RetrieveRecordsSummary, undefined>;
30
27
  upsertRecords?(options: UpsertRecordsOptions): Promise<void>;
31
28
  }
32
29
  export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
@@ -53,13 +50,13 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
53
50
  byteLimit: number | undefined;
54
51
  chunkSize: number | undefined;
55
52
  dataFormatId: DataFormatId;
56
- encodingId: string;
53
+ encodingId: string | undefined;
57
54
  hasHeaders: boolean | undefined;
58
55
  parserId: string | undefined;
59
56
  path: string;
60
57
  stopSignal: AbortSignal | undefined;
61
58
  supportsTransferableStreams: boolean;
62
- valueDelimiterId: ValueDelimiterId;
59
+ valueDelimiterId: ValueDelimiterId | undefined;
63
60
  }
64
61
  export interface AuditObjectContentResult {
65
62
  contentAuditConfig: ContentAuditConfig;
@@ -67,8 +64,10 @@ export interface AuditObjectContentResult {
67
64
  }
68
65
  export type CloseStoreOptions = EngineConnectorActionOptions;
69
66
  export interface CreateObjectOptions extends EngineConnectorActionOptions {
67
+ columnConfigs: ObjectColumnConfig[];
68
+ indexedColumnIndexes: number[];
69
+ keyColumnIndex: number | undefined;
70
70
  path: string;
71
- structure: string;
72
71
  }
73
72
  export type DescribeConnectionOptions = EngineConnectorActionOptions;
74
73
  interface DescribeConnectionResult {
@@ -91,15 +90,15 @@ export interface GetInfoResult {
91
90
  info: Record<string, unknown>;
92
91
  }
93
92
  export interface GetReadableStreamOptions extends EngineConnectorActionOptions {
94
- id: string;
95
93
  path: string;
96
94
  }
97
95
  export interface GetRecordOptions extends EngineConnectorActionOptions {
98
- id: string;
96
+ id: RecordKey;
99
97
  path: string;
100
98
  }
99
+ export type RecordKey = number | string;
101
100
  export interface GetRecordResult {
102
- record?: string[] | Record<string, unknown>;
101
+ record: StoreRecord;
103
102
  }
104
103
  export interface ListNodesOptions extends EngineConnectorActionOptions {
105
104
  folderPath: string;
@@ -193,44 +192,38 @@ export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptio
193
192
  }
194
193
  export type PreviewValueStateId = 'blank' | 'empty' | 'invalid' | 'null' | 'valid';
195
194
  export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
196
- keys: string[];
195
+ keys: RecordKey[];
197
196
  path: string;
198
197
  }
199
- export interface RetrieveChunksOptions extends EngineConnectorActionOptions {
200
- chunkSize: number | undefined;
201
- encodingId: string;
202
- path: string;
203
- valueDelimiterId: ValueDelimiterId;
204
- }
205
198
  export interface RetrieveRecordsOptions extends EngineConnectorActionOptions {
206
199
  chunkSize: number | undefined;
207
200
  dataFormatId: DataFormatId;
208
- encodingId: string;
201
+ encodingId: string | undefined;
209
202
  parserId: string | undefined;
210
203
  path: string;
211
- range?: RecordRange | undefined;
212
- valueDelimiterId: ValueDelimiterId;
213
- }
214
- export interface RecordRange {
215
- limit: number;
216
- offset: number;
204
+ valueDelimiterId: ValueDelimiterId | undefined;
217
205
  }
218
206
  export interface RetrieveRecordsSummary {
219
- byteCount: number;
220
- commentLineCount: number;
221
- emptyLineCount: number;
222
- lineCount: number;
223
- nonUniformRecordCount: number;
224
207
  recordCount: number;
225
- totalCount?: number | undefined;
226
208
  }
227
- export type RecordRetrievalTypeId = 'jsonRecordArray' | 'parsingRecordArray';
209
+ export type RecordRetrievalTypeId = 'parsingRecordArray' | 'storeRecordArray';
210
+ export type RetrieveRecordsChunk = ParsedRecordsChunk | {
211
+ records: StoreRecord[];
212
+ typeId: 'storeRecordArray';
213
+ };
214
+ export interface ParsedRecordsChunk {
215
+ records: ParsedRecord[];
216
+ typeId: 'parsingRecordArray';
217
+ }
228
218
  export interface UpsertRecordsOptions extends EngineConnectorActionOptions {
229
- records: Record<string, unknown>[];
219
+ records: StoreRecord[];
230
220
  path: string;
231
221
  }
232
222
  export type ConnectorUsageId = InferOutput<typeof connectorUsageIdSchema>;
233
223
  export { auditObjectContentWithParser, buildPreviewObjectResult, retrieveRecordsWithParser } from './objectParsing';
224
+ export { resolveStorePath } from './storePath';
225
+ export { convertStoreRecordToParsedRecord, convertStoreValueToText } from './storeValues';
226
+ export { acquireWebLock } from './webLock';
234
227
  export { readTableSchema } from './tableSchema';
235
228
  export declare function determineConnectorUsageId(actionNames: ConnectorActionName[]): ConnectorUsageId;
236
229
  export declare const constructConnectorCategoryConfig: (id: string, localeId?: import('../../../locale').LocaleId) => LocalisedConfig<ConnectorCategoryConfig>;
@@ -1,5 +1,5 @@
1
- import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
2
- import { DeclaredColumnConfig, DeclaredReadingConfig, ParsedRecord, ReadingChoiceConfig } from '../../dataView';
1
+ import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, ParsedRecordsChunk, PreviewObjectResult, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
2
+ import { DeclaredColumnConfig, DeclaredReadingConfig, ReadingChoiceConfig } from '../../dataView';
3
3
  import { FilePreviewResult, ToolConfig } from '../tool';
4
4
  export declare function buildPreviewObjectResult(input: {
5
5
  asAt: number;
@@ -12,12 +12,11 @@ export declare function buildPreviewObjectResult(input: {
12
12
  toolConfigs: ToolConfig[];
13
13
  }): Promise<PreviewObjectResult>;
14
14
  export declare function retrieveRecordsWithParser(input: {
15
- chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>;
16
15
  options: RetrieveRecordsOptions;
17
16
  signal: AbortSignal;
18
17
  stream: ReadableStream<Uint8Array>;
19
18
  toolConfigs: ToolConfig[];
20
- }): Promise<RetrieveRecordsSummary>;
19
+ }): AsyncGenerator<ParsedRecordsChunk, RetrieveRecordsSummary, undefined>;
21
20
  export declare function auditObjectContentWithParser(input: {
22
21
  constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
23
22
  declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
@@ -0,0 +1,4 @@
1
+ export declare function resolveStorePath(path: string): {
2
+ databaseName: string | undefined;
3
+ tableName: string;
4
+ };
@@ -0,0 +1,4 @@
1
+ import { ParsedRecord } from '../../dataView';
2
+ import { StoreRecord, StoreValue } from '.';
3
+ export declare function convertStoreRecordToParsedRecord(storeRecord: StoreRecord): ParsedRecord;
4
+ export declare function convertStoreValueToText(storeValue: StoreValue): string | null;
@@ -0,0 +1 @@
1
+ export declare function acquireWebLock(name: string): Promise<(() => void) | undefined>;
@@ -25,7 +25,9 @@ export interface EngineAuthActionOptions {
25
25
  export interface EngineConnectorActionOptions {
26
26
  accountId?: string;
27
27
  appCheckToken?: string;
28
+ requestId?: string;
28
29
  sessionAccessToken?: string;
30
+ signal?: AbortSignal | undefined;
29
31
  }
30
32
  export interface EngineContextActionOptions {
31
33
  accountId?: string;
@@ -39,3 +41,4 @@ export interface EngineCallbackData {
39
41
  export interface EngineConfig extends ModuleConfig {
40
42
  typeId: 'engine';
41
43
  }
44
+ export declare function resolveRequestSignal(options: Pick<EngineConnectorActionOptions, 'signal'>): AbortSignal;
@@ -5,7 +5,7 @@ export { toolConfigSchema } from './toolConfig.schema';
5
5
  export type ToolConfig = InferOutput<typeof toolConfigSchema>;
6
6
  export type ParserConfig = InferOutput<typeof parserConfigSchema>;
7
7
  export type { FilePreviewerInterface, FilePreviewResult } from './filePreviewer';
8
- export type { ParserInterface, ParseStreamOptions, ParseTextOptions, ParseTextResult } from './parser';
8
+ export type { ParserInterface, ParseStreamOptions, ParseStreamSummary, ParseTextOptions, ParseTextResult } from './parser';
9
9
  export { parseDelimitedText, type ParseTextWithDelimiter } from './delimitedText';
10
10
  export declare const TOOL_NAME_PREFIX = "dpuse-tool-";
11
11
  export declare function listParserToolConfigs(toolConfigs: ToolConfig[], dataFormatId: DataFormatId): ToolConfig[];
@@ -1,7 +1,7 @@
1
1
  import { DataFormatId, ParsedRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
2
- import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
2
+ import { ParsedRecordsChunk, RetrieveRecordsSummary } from '../connector';
3
3
  export interface ParserInterface {
4
- parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
4
+ parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions) => AsyncGenerator<ParsedRecordsChunk, ParseStreamSummary, undefined>;
5
5
  parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
6
6
  }
7
7
  export interface ParseTextOptions {
@@ -22,3 +22,10 @@ export interface ParseStreamOptions {
22
22
  encodingId: string;
23
23
  valueDelimiterId: ValueDelimiterId;
24
24
  }
25
+ export interface ParseStreamSummary extends RetrieveRecordsSummary {
26
+ byteCount: number;
27
+ commentLineCount: number;
28
+ emptyLineCount: number;
29
+ lineCount: number;
30
+ nonUniformRecordCount: number;
31
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.210",
3
+ "version": "1.0.212",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",