@3sixty/data-processing-models 0.0.0-stage → 0.20.0-next.0

package/dist/types/models/IRule.d.ts

@@ -0,0 +1,23 @@
+import type { CoerceType } from "@3sixty/core";
+/**
+ * Rule defining how to extract data from an object.
+ */
+export interface IRule {
+    /**
+     * The JSONPath expression identifying the data to extract from the document.
+     * @see https://www.rfc-editor.org/rfc/rfc9535.html
+     */
+    source: string;
+    /**
+     * The target path of where to store the extracted data, using dotted or numeric index notation.
+     */
+    target: string;
+    /**
+     * When extracting objects, how much of the original path should be maintained in the target object.
+     */
+    retainPathDepth?: number;
+    /**
+     * Should the data be coerced to a specific type.
+     */
+    coerce?: CoerceType;
+}

package/dist/types/models/IRuleGroup.d.ts

@@ -0,0 +1,18 @@
+import type { IRule } from "./IRule.js";
+/**
+ * Group of rules for extraction.
+ */
+export interface IRuleGroup {
+    /**
+     * The id for the rule group.
+     */
+    id: string;
+    /**
+     * The label for the rule group.
+     */
+    label: string;
+    /**
+     * The rules.
+     */
+    rules: IRule[];
+}

package/dist/types/models/IStructuredData.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Definition for the structured data.
+ */
+export interface IStructuredData {
+    /**
+     * The MIME type of the data.
+     */
+    sourceMimeType: string;
+    /**
+     * The structured object.
+     */
+    object: unknown;
+}

package/dist/types/models/api/IDataProcessingConvertRequest.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Perform a conversion on the data.
+ */
+export interface IDataProcessingConvertRequest {
+    /**
+     * The request body containing the data to convert.
+     */
+    body: {
+        /**
+         * The binary data to convert in base64.
+         */
+        data: string;
+        /**
+         * Use the specified mime type for conversion, will auto detect if undefined.
+         */
+        overrideMimeType?: string;
+    };
+}

package/dist/types/models/api/IDataProcessingConvertResponse.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Response to converting data.
+ */
+export interface IDataProcessingConvertResponse {
+    /**
+     * The converted data as an object.
+     */
+    body: unknown;
+}

package/dist/types/models/api/IDataProcessingExtractRequest.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Perform an extraction on the data with the specified ruleset.
+ */
+export interface IDataProcessingExtractRequest {
+    /**
+     * The request body containing the data and rule group to use for extraction.
+     */
+    body: {
+        /**
+         * The rule group id to use for the extraction.
+         */
+        ruleGroupId: string;
+        /**
+         * The binary data to extract from in base64.
+         */
+        data: string;
+        /**
+         * The default extractor connector will be used if not specified.
+         */
+        overrideExtractorType?: string;
+        /**
+         * An optional override for the mime type for conversion, will auto detect if not provided.
+         */
+        overrideMimeType?: string;
+    };
+}

package/dist/types/models/api/IDataProcessingExtractResponse.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Response to extracting data.
+ */
+export interface IDataProcessingExtractResponse {
+    /**
+     * The extracted data in extended JSON format, preserving types such as bigint, dates, and Uint8Array.
+     */
+    body: unknown;
+}

package/dist/types/models/api/IDataProcessingRuleGroupGetRequest.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Get a rule group.
+ */
+export interface IDataProcessingRuleGroupGetRequest {
+    /**
+     * The path parameters for the request.
+     */
+    pathParams: {
+        /**
+         * The id of the rule group to retrieve.
+         */
+        id: string;
+    };
+}

package/dist/types/models/api/IDataProcessingRuleGroupGetResponse.d.ts

@@ -0,0 +1,10 @@
+import type { IRuleGroup } from "../IRuleGroup.js";
+/**
+ * Get a rule group response.
+ */
+export interface IDataProcessingRuleGroupGetResponse {
+    /**
+     * The rule group data.
+     */
+    body: IRuleGroup;
+}

package/dist/types/models/api/IDataProcessingRuleGroupListRequest.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Get a list of the rule group entries.
+ */
+export interface IDataProcessingRuleGroupListRequest {
+    /**
+     * The query parameters.
+     */
+    query?: {
+        /**
+         * The optional cursor to get next chunk.
+         */
+        cursor?: string;
+        /**
+         * Limit the number of entities to return.
+         */
+        limit?: string;
+    };
+}

package/dist/types/models/api/IDataProcessingRuleGroupListResponse.d.ts

@@ -0,0 +1,19 @@
+import type { IRuleGroup } from "../IRuleGroup.js";
+/**
+ * Response for rule group entry list request.
+ */
+export interface IDataProcessingRuleGroupListResponse {
+    /**
+     * The response body containing the list of rule groups and optional pagination cursor.
+     */
+    body: {
+        /**
+         * The entities, which can be partial if a limited keys list was provided.
+         */
+        entities: IRuleGroup[];
+        /**
+         * An optional cursor, when defined can be used to call find to get more entities.
+         */
+        cursor?: string;
+    };
+}

package/dist/types/models/api/IDataProcessingRuleGroupRemoveRequest.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Remove a rule group.
+ */
+export interface IDataProcessingRuleGroupRemoveRequest {
+    /**
+     * The path parameters for the request.
+     */
+    pathParams: {
+        /**
+         * The id of the rule group to remove.
+         */
+        id: string;
+    };
+}

package/dist/types/models/api/IDataProcessingRuleGroupSetRequest.d.ts

@@ -0,0 +1,28 @@
+import type { IRule } from "../IRule.js";
+/**
+ * Set a rule group.
+ */
+export interface IDataProcessingRuleGroupSetRequest {
+    /**
+     * The path parameters for the request.
+     */
+    pathParams: {
+        /**
+         * The id of the rule group to set.
+         */
+        id: string;
+    };
+    /**
+     * The rule group data to store.
+     */
+    body: {
+        /**
+         * The label for the rule group.
+         */
+        label: string;
+        /**
+         * The rules.
+         */
+        rules: IRule[];
+    };
+}

package/docs/examples.md

@@ -0,0 +1,35 @@
+# Data Processing Models Examples
+
+These examples focus on registering and resolving connector implementations so processing components can discover them at runtime.
+
+## DataConverterConnectorFactory
+
+```typescript
+import { JsonConverterConnector } from '@3sixty/data-processing-converters';
+import { DataConverterConnectorFactory } from '@3sixty/data-processing-models';
+
+DataConverterConnectorFactory.register('json', () => new JsonConverterConnector());
+
+const converterNames = DataConverterConnectorFactory.names();
+const converter = DataConverterConnectorFactory.get('json');
+const converterInstances = DataConverterConnectorFactory.instancesList();
+
+console.log(converterNames); // ['json']
+console.log(converter.className()); // jsonConverterConnector
+console.log(converterInstances.length); // 1
+```
+
+## DataExtractorConnectorFactory
+
+```typescript
+import { JsonPathExtractorConnector } from '@3sixty/data-processing-extractors';
+import { DataExtractorConnectorFactory } from '@3sixty/data-processing-models';
+
+DataExtractorConnectorFactory.register('JSONPath', () => new JsonPathExtractorConnector());
+
+const extractorNames = DataExtractorConnectorFactory.names();
+const extractor = DataExtractorConnectorFactory.get('JSONPath');
+
+console.log(extractorNames); // ['JSONPath']
+console.log(extractor.className()); // jsonPathExtractorConnector
+```

package/docs/reference/index.md

@@ -0,0 +1,24 @@
+# @3sixty/data-processing-models
+
+## Interfaces
+
+- [IDataConverterConnector](interfaces/IDataConverterConnector.md)
+- [IDataExtractorConnector](interfaces/IDataExtractorConnector.md)
+- [IDataProcessingComponent](interfaces/IDataProcessingComponent.md)
+- [IRule](interfaces/IRule.md)
+- [IRuleGroup](interfaces/IRuleGroup.md)
+- [IDataProcessingConvertRequest](interfaces/IDataProcessingConvertRequest.md)
+- [IDataProcessingConvertResponse](interfaces/IDataProcessingConvertResponse.md)
+- [IDataProcessingExtractRequest](interfaces/IDataProcessingExtractRequest.md)
+- [IDataProcessingExtractResponse](interfaces/IDataProcessingExtractResponse.md)
+- [IDataProcessingRuleGroupGetRequest](interfaces/IDataProcessingRuleGroupGetRequest.md)
+- [IDataProcessingRuleGroupGetResponse](interfaces/IDataProcessingRuleGroupGetResponse.md)
+- [IDataProcessingRuleGroupListRequest](interfaces/IDataProcessingRuleGroupListRequest.md)
+- [IDataProcessingRuleGroupListResponse](interfaces/IDataProcessingRuleGroupListResponse.md)
+- [IDataProcessingRuleGroupRemoveRequest](interfaces/IDataProcessingRuleGroupRemoveRequest.md)
+- [IDataProcessingRuleGroupSetRequest](interfaces/IDataProcessingRuleGroupSetRequest.md)
+
+## Variables
+
+- [DataConverterConnectorFactory](variables/DataConverterConnectorFactory.md)
+- [DataExtractorConnectorFactory](variables/DataExtractorConnectorFactory.md)

package/docs/reference/interfaces/IDataConverterConnector.md

@@ -0,0 +1,43 @@
+# Interface: IDataConverterConnector
+
+Interface describing a connector for converting data.
+
+## Extends
+
+- `IComponent`
+
+## Methods
+
+### mimeTypes() {#mimetypes}
+
+> **mimeTypes**(): `string`[]
+
+Returns the MIME types that this connector can convert.
+
+#### Returns
+
+`string`[]
+
+The supported MIME type strings.
+
+***
+
+### convert() {#convert}
+
+> **convert**(`data`): `Promise`\<`unknown`\>
+
+Converts the data to a structured object.
+
+#### Parameters
+
+##### data
+
+`Uint8Array`
+
+The binary data to convert.
+
+#### Returns
+
+`Promise`\<`unknown`\>
+
+The parsed structured object.

package/docs/reference/interfaces/IDataExtractorConnector.md

@@ -0,0 +1,35 @@
+# Interface: IDataExtractorConnector
+
+Interface describing a connector for extracting data.
+
+## Extends
+
+- `IComponent`
+
+## Methods
+
+### extract() {#extract}
+
+> **extract**(`data`, `rules`): `Promise`\<`unknown`\>
+
+Extracts data from the provided input.
+
+#### Parameters
+
+##### data
+
+`unknown`
+
+The object to extract from.
+
+##### rules
+
+[`IRule`](IRule.md)[]
+
+The rules to use to extract the data.
+
+#### Returns
+
+`Promise`\<`unknown`\>
+
+The extracted data.

package/docs/reference/interfaces/IDataProcessingComponent.md

@@ -0,0 +1,170 @@
+# Interface: IDataProcessingComponent
+
+Interface describing a component for processing data.
+
+## Extends
+
+- `IComponent`
+
+## Methods
+
+### ruleGroupSet() {#rulegroupset}
+
+> **ruleGroupSet**(`ruleGroup`): `Promise`\<`void`\>
+
+Set an extraction rule group.
+
+#### Parameters
+
+##### ruleGroup
+
+[`IRuleGroup`](IRuleGroup.md)
+
+The rule group to store.
+
+#### Returns
+
+`Promise`\<`void`\>
+
+A promise that resolves when the rule group has been stored.
+
+***
+
+### ruleGroupGet() {#rulegroupget}
+
+> **ruleGroupGet**(`ruleGroupId`): `Promise`\<[`IRuleGroup`](IRuleGroup.md)\>
+
+Get a rule group for extraction.
+
+#### Parameters
+
+##### ruleGroupId
+
+`string`
+
+The id of the rule group to get.
+
+#### Returns
+
+`Promise`\<[`IRuleGroup`](IRuleGroup.md)\>
+
+The rule group.
+
+***
+
+### ruleGroupRemove() {#rulegroupremove}
+
+> **ruleGroupRemove**(`ruleGroupId`): `Promise`\<`void`\>
+
+Remove a rule group.
+
+#### Parameters
+
+##### ruleGroupId
+
+`string`
+
+The id of the rule group to remove.
+
+#### Returns
+
+`Promise`\<`void`\>
+
+A promise that resolves when the rule group has been removed.
+
+***
+
+### extract() {#extract}
+
+> **extract**(`ruleGroupId`, `data`, `overrideExtractorType?`, `overrideMimeType?`): `Promise`\<`unknown`\>
+
+Extracts data from the provided input.
+
+#### Parameters
+
+##### ruleGroupId
+
+`string`
+
+The id of the rule group to use to extract data.
+
+##### data
+
+`Uint8Array`
+
+The data to extract from.
+
+##### overrideExtractorType?
+
+`string`
+
+An optional override for the extractor type.
+
+##### overrideMimeType?
+
+`string`
+
+An optional override for the mime type for conversion, will auto detect if not provided.
+
+#### Returns
+
+`Promise`\<`unknown`\>
+
+The extracted data.
+
+***
+
+### convert() {#convert}
+
+> **convert**(`data`, `overrideMimeType?`): `Promise`\<`unknown`\>
+
+Converts data from the provided input to a structured JSON document.
+
+#### Parameters
+
+##### data
+
+`Uint8Array`
+
+The data to convert.
+
+##### overrideMimeType?
+
+`string`
+
+An optional override for the mime type, will auto detect if empty.
+
+#### Returns
+
+`Promise`\<`unknown`\>
+
+The converted data.
+
+***
+
+### query() {#query}
+
+> **query**(`cursor?`, `limit?`): `Promise`\<\{ `entities`: [`IRuleGroup`](IRuleGroup.md)[]; `cursor?`: `string`; \}\>
+
+Query the rule group entries.
+
+#### Parameters
+
+##### cursor?
+
+`string`
+
+The cursor to request the next chunk of entities.
+
+##### limit?
+
+`number`
+
+Limit the number of entities to return.
+
+#### Returns
+
+`Promise`\<\{ `entities`: [`IRuleGroup`](IRuleGroup.md)[]; `cursor?`: `string`; \}\>
+
+All the entities for the storage matching the conditions,
+and a cursor which can be used to request more entities.

package/docs/reference/interfaces/IDataProcessingConvertRequest.md

@@ -0,0 +1,23 @@
+# Interface: IDataProcessingConvertRequest
+
+Perform a conversion on the data.
+
+## Properties
+
+### body {#body}
+
+> **body**: `object`
+
+The request body containing the data to convert.
+
+#### data
+
+> **data**: `string`
+
+The binary data to convert in base64.
+
+#### overrideMimeType?
+
+> `optional` **overrideMimeType?**: `string`
+
+Use the specified mime type for conversion, will auto detect if undefined.

package/docs/reference/interfaces/IDataProcessingConvertResponse.md

@@ -0,0 +1,11 @@
+# Interface: IDataProcessingConvertResponse
+
+Response to converting data.
+
+## Properties
+
+### body {#body}
+
+> **body**: `unknown`
+
+The converted data as an object.

package/docs/reference/interfaces/IDataProcessingExtractRequest.md

@@ -0,0 +1,35 @@
+# Interface: IDataProcessingExtractRequest
+
+Perform an extraction on the data with the specified ruleset.
+
+## Properties
+
+### body {#body}
+
+> **body**: `object`
+
+The request body containing the data and rule group to use for extraction.
+
+#### ruleGroupId
+
+> **ruleGroupId**: `string`
+
+The rule group id to use for the extraction.
+
+#### data
+
+> **data**: `string`
+
+The binary data to extract from in base64.
+
+#### overrideExtractorType?
+
+> `optional` **overrideExtractorType?**: `string`
+
+The default extractor connector will be used if not specified.
+
+#### overrideMimeType?
+
+> `optional` **overrideMimeType?**: `string`
+
+An optional override for the mime type for conversion, will auto detect if not provided.

package/docs/reference/interfaces/IDataProcessingExtractResponse.md

@@ -0,0 +1,11 @@
+# Interface: IDataProcessingExtractResponse
+
+Response to extracting data.
+
+## Properties
+
+### body {#body}
+
+> **body**: `unknown`
+
+The extracted data in extended JSON format, preserving types such as bigint, dates, and Uint8Array.

package/docs/reference/interfaces/IDataProcessingRuleGroupGetRequest.md

@@ -0,0 +1,17 @@
+# Interface: IDataProcessingRuleGroupGetRequest
+
+Get a rule group.
+
+## Properties
+
+### pathParams {#pathparams}
+
+> **pathParams**: `object`
+
+The path parameters for the request.
+
+#### id
+
+> **id**: `string`
+
+The id of the rule group to retrieve.

package/docs/reference/interfaces/IDataProcessingRuleGroupGetResponse.md

@@ -0,0 +1,11 @@
+# Interface: IDataProcessingRuleGroupGetResponse
+
+Get a rule group response.
+
+## Properties
+
+### body {#body}
+
+> **body**: [`IRuleGroup`](IRuleGroup.md)
+
+The rule group data.

package/docs/reference/interfaces/IDataProcessingRuleGroupListRequest.md

@@ -0,0 +1,23 @@
+# Interface: IDataProcessingRuleGroupListRequest
+
+Get a list of the rule group entries.
+
+## Properties
+
+### query? {#query}
+
+> `optional` **query?**: `object`
+
+The query parameters.
+
+#### cursor?
+
+> `optional` **cursor?**: `string`
+
+The optional cursor to get next chunk.
+
+#### limit?
+
+> `optional` **limit?**: `string`
+
+Limit the number of entities to return.