@dpuse/dpuse-shared 1.0.200 → 1.0.203

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -47,7 +47,5 @@ export interface ObjectColumnConfig {
47
47
  label: LocaleLabel;
48
48
  dataTypeId: DataTypeId;
49
49
  dataSubtypeId: DataSubtypeId | undefined;
50
- storageTypeId: StorageTypeId | undefined;
51
50
  }
52
- export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
53
51
  export {};
@@ -17,6 +17,7 @@ export interface ReadingConfig {
17
17
  encodingId: string | undefined;
18
18
  fileType: FileTypeResult | undefined;
19
19
  hasHeaders: boolean | undefined;
20
+ isDeclared?: boolean | undefined;
20
21
  parserId: string | undefined;
21
22
  recordDelimiterId: RecordDelimiterId | undefined;
22
23
  valueDelimiterId: ValueDelimiterId | undefined;
@@ -69,9 +70,10 @@ export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
69
70
  export interface ColumnAuditConfig extends ColumnReadingConfig {
70
71
  inferenceCounts: Record<string, number>;
71
72
  leadingInferenceCounts: Record<string, number>;
73
+ isTypeDeclared?: boolean | undefined;
72
74
  numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
73
75
  dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
74
- isFormatAmbiguous: boolean | undefined;
76
+ formatIssueIds: FormatIssueId[] | undefined;
75
77
  nullWordValueCount?: number | undefined;
76
78
  validValueCount: number | undefined;
77
79
  invalidValueCount: number | undefined;
@@ -99,8 +101,21 @@ export interface ColumnAuditConfig extends ColumnReadingConfig {
99
101
  validValueBinScaleId?: ValueBinScaleId | undefined;
100
102
  validValueBinOutliers?: ValueBinOutlierConfig | undefined;
101
103
  }
102
- export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
104
+ export interface DeclaredColumnConfig {
105
+ name: string;
106
+ label: string | undefined;
107
+ dataTypeId: Exclude<DataTypeId, 'unknown'> | undefined;
108
+ dataSubtypeId: DataSubtypeId | undefined;
109
+ }
110
+ export interface DeclaredReadingConfig {
111
+ dataFormatId: DataFormatId;
112
+ encodingId: string | undefined;
113
+ hasHeaders: boolean | undefined;
114
+ valueDelimiterId: ValueDelimiterId | undefined;
115
+ }
116
+ export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits'>;
103
117
  export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
118
+ export type FormatIssueId = 'booleanPairMixed' | 'currencyMixed' | 'dateOrderMixed' | 'dateOrderUnproven' | 'decimalSeparatorMixed' | 'decimalSeparatorUnproven' | 'decimalSeparatorWeaklyProven' | 'percentageMixed' | 'scientificNotationMixed';
104
119
  export interface ValueBinConfig {
105
120
  count: number;
106
121
  lowerValue: string;
@@ -2,11 +2,10 @@ import { InferOutput } from 'valibot';
2
2
  import { EngineConnectorActionOptions } from '../engine';
3
3
  import { LocalisedConfig } from '../../../locale';
4
4
  import { ToolConfig } from '../tool';
5
- import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
5
+ import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, DeclaredColumnConfig, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
6
6
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
7
7
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
8
8
  export { connectorConfigSchema } from './connectorConfig.schema';
9
- export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
10
9
  export interface ConnectorInterface {
11
10
  abortOperation?(): void;
12
11
  auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
@@ -32,9 +31,9 @@ export interface ConnectorInterface {
32
31
  export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
33
32
  export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities, toolConfigs: ToolConfig[]) => ConnectorInterface;
34
33
  export interface ConnectorUtilities {
35
- constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
34
+ constructColumnAuditor: (hasHeaders: boolean | undefined, declaredColumnConfigs?: DeclaredColumnConfig[]) => ColumnAuditor;
36
35
  hasReadableStreamTransferSupport(): boolean;
37
- inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
36
+ inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean, declaredColumnConfigs?: DeclaredColumnConfig[]) => InferenceSummary;
38
37
  }
39
38
  export interface ColumnAuditor {
40
39
  auditRecords: (parsedRecords: ParsedRecord[]) => void;
@@ -229,6 +228,8 @@ export interface UpsertRecordsOptions extends EngineConnectorActionOptions {
229
228
  path: string;
230
229
  }
231
230
  export type ConnectorUsageId = InferOutput<typeof connectorUsageIdSchema>;
231
+ export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
232
+ export { readTableSchema } from './tableSchema';
232
233
  export declare function determineConnectorUsageId(actionNames: ConnectorActionName[]): ConnectorUsageId;
233
234
  export declare const constructConnectorCategoryConfig: (id: string, localeId?: import('../../../locale').LocaleId) => LocalisedConfig<ConnectorCategoryConfig>;
234
235
  export declare const constructConnectorItemTypeConfig: (id: string, localeId?: import('../../../locale').LocaleId) => LocalisedConfig<ConnectorItemTypeConfig>;
@@ -1,6 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
- import { AuditObjectContentOptions, AuditObjectContentResult, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
- import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
2
+ import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
+ import { DataFormatId, DeclaredColumnConfig, DeclaredReadingConfig, ParsedRecord, ValueDelimiterId } from '../../dataView';
4
4
  import { ToolConfig } from '../tool';
5
5
  export interface ObjectFilePreview {
6
6
  bytes: Uint8Array;
@@ -13,6 +13,8 @@ export interface ObjectFilePreview {
13
13
  }
14
14
  export declare function buildPreviewObjectResult(input: {
15
15
  asAt: number;
16
+ declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
17
+ declaredReadingConfig?: DeclaredReadingConfig | undefined;
16
18
  filePreview: ObjectFilePreview;
17
19
  inferDataTypes: ConnectorUtilities['inferDataTypes'];
18
20
  parserId: string | undefined;
@@ -28,7 +30,8 @@ export declare function retrieveRecordsWithParser(input: {
28
30
  toolConfigs: ToolConfig[];
29
31
  }): Promise<RetrieveRecordsSummary>;
30
32
  export declare function auditObjectContentWithParser(input: {
31
- constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
33
+ constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
34
+ declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
32
35
  onProgress: (recordCount: number, byteCount: number) => void;
33
36
  options: AuditObjectContentOptions;
34
37
  signal: AbortSignal;
@@ -0,0 +1,9 @@
1
+ import { DeclaredColumnConfig, DeclaredReadingConfig } from '../../dataView';
2
+ export declare function readTableSchema(resource: {
3
+ dialect?: unknown;
4
+ encoding?: string | undefined;
5
+ schema: unknown;
6
+ }): {
7
+ declaredColumnConfigs: DeclaredColumnConfig[];
8
+ declaredReadingConfig: DeclaredReadingConfig;
9
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.200",
3
+ "version": "1.0.203",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",
@@ -64,12 +64,12 @@
64
64
  "valibot": "^1.5.0"
65
65
  },
66
66
  "devDependencies": {
67
- "@dpuse/dpuse-development": "^1.0.148",
68
- "@dpuse/eslint-config-dpuse": "^1.0.149",
67
+ "@dpuse/dpuse-development": "^1.0.149",
68
+ "@dpuse/eslint-config-dpuse": "^1.0.150",
69
69
  "@types/node": "^26.6.6",
70
70
  "@vitest/coverage-v8": "^5.1.0",
71
71
  "eslint": "^10.14.0",
72
- "fallow": "^3.41.0",
72
+ "fallow": "^3.42.0",
73
73
  "fast-check": "^4.10.2",
74
74
  "iconv-lite": "^0.7.3",
75
75
  "jschardet": "^4.1.0",