@dpuse/dpuse-shared 1.0.199 → 1.0.202

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -45,16 +45,9 @@ export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
45
45
  }
46
46
  export interface ObjectColumnConfig {
47
47
  label: LocaleLabel;
48
- derivedFromColumnIndex?: number | undefined;
49
48
  dataTypeId: DataTypeId;
50
49
  dataSubtypeId: DataSubtypeId | undefined;
51
50
  storageTypeId: StorageTypeId | undefined;
52
- decimalSeparatorId: DecimalSeparatorId | undefined;
53
- dateOrderId: DateOrderId | undefined;
54
- nullWordsAreEmpty?: boolean | undefined;
55
51
  }
56
- export type DateOrderId = 'dayFirst' | 'monthFirst';
57
- export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
58
52
  export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
59
- export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
60
53
  export {};
@@ -1,6 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
2
  import { ComponentInstanceConfig } from '..';
3
- import { ConnectionNodeConfig, DateOrderId, DecimalSeparatorId, ObjectColumnConfig } from '../connection';
3
+ import { ConnectionNodeConfig, ObjectColumnConfig } from '../connection';
4
4
  export interface DataViewConfig extends ComponentInstanceConfig {
5
5
  typeId: 'dataView';
6
6
  connectionId: string;
@@ -58,9 +58,18 @@ export interface ContentAuditConfig {
58
58
  recordCount: number;
59
59
  sourceWaitDuration: number | undefined;
60
60
  }
61
- export interface ColumnAuditConfig extends ObjectColumnConfig {
61
+ export interface ColumnReadingConfig extends ObjectColumnConfig {
62
+ derivedFromColumnIndex?: number | undefined;
63
+ decimalSeparatorId: DecimalSeparatorId | undefined;
64
+ dateOrderId: DateOrderId | undefined;
65
+ nullWordsAreEmpty?: boolean | undefined;
66
+ }
67
+ export type DateOrderId = 'dayFirst' | 'monthFirst';
68
+ export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
69
+ export interface ColumnAuditConfig extends ColumnReadingConfig {
62
70
  inferenceCounts: Record<string, number>;
63
71
  leadingInferenceCounts: Record<string, number>;
72
+ isTypeDeclared?: boolean | undefined;
64
73
  numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
65
74
  dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
66
75
  isFormatAmbiguous: boolean | undefined;
@@ -91,6 +100,18 @@ export interface ColumnAuditConfig extends ObjectColumnConfig {
91
100
  validValueBinScaleId?: ValueBinScaleId | undefined;
92
101
  validValueBinOutliers?: ValueBinOutlierConfig | undefined;
93
102
  }
103
+ export interface DeclaredColumnConfig {
104
+ name: string;
105
+ label: string | undefined;
106
+ dataTypeId: Exclude<DataTypeId, 'unknown'> | undefined;
107
+ dataSubtypeId: DataSubtypeId | undefined;
108
+ }
109
+ export interface DeclaredReadingConfig {
110
+ dataFormatId: DataFormatId;
111
+ encodingId: string | undefined;
112
+ hasHeaders: boolean | undefined;
113
+ valueDelimiterId: ValueDelimiterId | undefined;
114
+ }
94
115
  export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
95
116
  export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
96
117
  export interface ValueBinConfig {
@@ -248,5 +269,6 @@ export declare const DATA_TYPE_CONFIGS: readonly [{
248
269
  readonly isChoosable: false;
249
270
  readonly subtypeIds: readonly [];
250
271
  }];
272
+ export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
251
273
  export declare function getValueDelimiterCharacter(valueDelimiterId: ValueDelimiterId): string;
252
274
  export {};
@@ -1,15 +1,14 @@
1
1
  import { InferOutput } from 'valibot';
2
- import { Component } from '../..';
3
2
  import { EngineConnectorActionOptions } from '../engine';
4
3
  import { LocalisedConfig } from '../../../locale';
5
4
  import { ToolConfig } from '../tool';
6
- import { ColumnAuditConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
5
+ import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, DeclaredColumnConfig, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
7
6
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
8
7
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
9
8
  export { connectorConfigSchema } from './connectorConfig.schema';
10
9
  export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
11
- export interface ConnectorInterface extends Component {
12
- readonly config: ConnectorConfig;
10
+ export { readTableSchema } from './tableSchema';
11
+ export interface ConnectorInterface {
13
12
  abortOperation?(): void;
14
13
  auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
15
14
  closeStore?(options: CloseStoreOptions): Promise<void>;
@@ -34,9 +33,9 @@ export interface ConnectorInterface extends Component {
34
33
  export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
35
34
  export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities, toolConfigs: ToolConfig[]) => ConnectorInterface;
36
35
  export interface ConnectorUtilities {
37
- constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
36
+ constructColumnAuditor: (hasHeaders: boolean | undefined, declaredColumnConfigs?: DeclaredColumnConfig[]) => ColumnAuditor;
38
37
  hasReadableStreamTransferSupport(): boolean;
39
- inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
38
+ inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean, declaredColumnConfigs?: DeclaredColumnConfig[]) => InferenceSummary;
40
39
  }
41
40
  export interface ColumnAuditor {
42
41
  auditRecords: (parsedRecords: ParsedRecord[]) => void;
@@ -123,6 +122,7 @@ export interface LoadObjectRecordsResult {
123
122
  recordCount: number;
124
123
  }
125
124
  export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
125
+ columnConfigs: ColumnReadingConfig[];
126
126
  hasHeaders: boolean;
127
127
  ignoredColumnIndexes: number[];
128
128
  headerValues: (string | null)[] | undefined;
@@ -188,7 +188,7 @@ export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptio
188
188
  parsedRecords: ParsedRecord[];
189
189
  }
190
190
  export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
191
- columnConfigs: ObjectColumnConfig[];
191
+ columnConfigs: ColumnReadingConfig[];
192
192
  parsedRecords: ParsedRecord[];
193
193
  }
194
194
  export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
@@ -1,6 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
- import { AuditObjectContentOptions, AuditObjectContentResult, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
- import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
2
+ import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
+ import { DataFormatId, DeclaredColumnConfig, ParsedRecord, ValueDelimiterId } from '../../dataView';
4
4
  import { ToolConfig } from '../tool';
5
5
  export interface ObjectFilePreview {
6
6
  bytes: Uint8Array;
@@ -13,7 +13,9 @@ export interface ObjectFilePreview {
13
13
  }
14
14
  export declare function buildPreviewObjectResult(input: {
15
15
  asAt: number;
16
+ declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
16
17
  filePreview: ObjectFilePreview;
18
+ hasHeaders?: boolean | undefined;
17
19
  inferDataTypes: ConnectorUtilities['inferDataTypes'];
18
20
  parserId: string | undefined;
19
21
  startedAt: number;
@@ -28,7 +30,8 @@ export declare function retrieveRecordsWithParser(input: {
28
30
  toolConfigs: ToolConfig[];
29
31
  }): Promise<RetrieveRecordsSummary>;
30
32
  export declare function auditObjectContentWithParser(input: {
31
- constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
33
+ constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
34
+ declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
32
35
  onProgress: (recordCount: number, byteCount: number) => void;
33
36
  options: AuditObjectContentOptions;
34
37
  signal: AbortSignal;
@@ -0,0 +1,9 @@
1
+ import { DeclaredColumnConfig, DeclaredReadingConfig } from '../../dataView';
2
+ export declare function readTableSchema(resource: {
3
+ dialect?: unknown;
4
+ encoding?: string | undefined;
5
+ schema: unknown;
6
+ }): {
7
+ declaredColumnConfigs: DeclaredColumnConfig[];
8
+ declaredReadingConfig: DeclaredReadingConfig;
9
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.199",
3
+ "version": "1.0.202",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",
@@ -64,12 +64,12 @@
64
64
  "valibot": "^1.5.0"
65
65
  },
66
66
  "devDependencies": {
67
- "@dpuse/dpuse-development": "^1.0.148",
68
- "@dpuse/eslint-config-dpuse": "^1.0.149",
67
+ "@dpuse/dpuse-development": "^1.0.149",
68
+ "@dpuse/eslint-config-dpuse": "^1.0.150",
69
69
  "@types/node": "^26.6.6",
70
70
  "@vitest/coverage-v8": "^5.1.0",
71
71
  "eslint": "^10.14.0",
72
- "fallow": "^3.41.0",
72
+ "fallow": "^3.42.0",
73
73
  "fast-check": "^4.10.2",
74
74
  "iconv-lite": "^0.7.3",
75
75
  "jschardet": "^4.1.0",