@3sixty/data-processing-service 0.0.0-stage → 0.20.0-next.1

package/dist/es/dataProcessingService.js

@@ -0,0 +1,175 @@
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+import { GeneralError, Guards, Is, NotFoundError, ObjectHelper } from "@3sixty/core";
+import { DataConverterConnectorFactory, DataExtractorConnectorFactory } from "@3sixty/data-processing-models";
+import { SortDirection } from "@3sixty/entity";
+import { EntityStorageConnectorFactory } from "@3sixty/entity-storage-models";
+import { MimeTypeHelper } from "@3sixty/web";
+/**
+ * Class for processing data from a source.
+ */
+export class DataProcessingService {
+    /**
+     * Runtime name for the class.
+     */
+    static CLASS_NAME = "DataProcessingService";
+    /**
+     * The entity storage for the extraction rule groups.
+     * @internal
+     */
+    _extractionRuleGroupStorage;
+    /**
+     * The extractor connection to use.
+     * @internal
+     */
+    _extractorConnector;
+    /**
+     * Create a new instance of DataExtractionService.
+     * @param options The options for the connector.
+     * @throws GeneralError If no extractor connector is registered.
+     */
+    constructor(options) {
+        this._extractionRuleGroupStorage = EntityStorageConnectorFactory.get(options?.extractionRuleGroupStorageConnectorType ?? "extraction-rule-group");
+        if (Is.stringValue(options?.defaultExtractorType)) {
+            this._extractorConnector = DataExtractorConnectorFactory.get(options.defaultExtractorType);
+        }
+        else {
+            const names = DataExtractorConnectorFactory.names();
+            if (names.length > 0) {
+                this._extractorConnector = DataExtractorConnectorFactory.get(names[0]);
+            }
+            else {
+                throw new GeneralError(DataProcessingService.CLASS_NAME, "noExtractorFound");
+            }
+        }
+    }
+    /**
+     * Returns the class name of the component.
+     * @returns The class name of the component.
+     */
+    className() {
+        return DataProcessingService.CLASS_NAME;
+    }
+    /**
+     * Set an extraction rule group.
+     * @param ruleGroup The rule group to store.
+     * @returns A promise that resolves when the rule group has been stored.
+     */
+    async ruleGroupSet(ruleGroup) {
+        Guards.object(DataProcessingService.CLASS_NAME, "ruleGroup", ruleGroup);
+        Guards.stringValue(DataProcessingService.CLASS_NAME, "ruleGroup.id", ruleGroup.id);
+        Guards.stringValue(DataProcessingService.CLASS_NAME, "ruleGroup.label", ruleGroup.label);
+        Guards.array(DataProcessingService.CLASS_NAME, "ruleGroup.rules", ruleGroup.rules);
+        for (const rule of ruleGroup.rules) {
+            Guards.stringValue(DataProcessingService.CLASS_NAME, "rule.source", rule.source);
+            Guards.stringValue(DataProcessingService.CLASS_NAME, "rule.target", rule.source);
+        }
+        const extractionRuleGroup = {
+            id: ruleGroup.id,
+            label: ruleGroup.label,
+            rules: ruleGroup.rules.map(rule => ({
+                source: rule.source,
+                target: rule.target,
+                retainPathDepth: rule.retainPathDepth,
+                coerce: rule.coerce
+            }))
+        };
+        await this._extractionRuleGroupStorage.set(extractionRuleGroup);
+    }
+    /**
+     * Get a rule group for extraction.
+     * @param ruleGroupId The id of the rule group to get.
+     * @returns The rule group.
+     */
+    async ruleGroupGet(ruleGroupId) {
+        Guards.stringValue(DataProcessingService.CLASS_NAME, "ruleGroupId", ruleGroupId);
+        const extractRuleGroup = await this._extractionRuleGroupStorage.get(ruleGroupId);
+        if (Is.empty(extractRuleGroup)) {
+            throw new NotFoundError(DataProcessingService.CLASS_NAME, "ruleGroupNotFound", ruleGroupId);
+        }
+        return {
+            id: extractRuleGroup.id,
+            label: extractRuleGroup.label,
+            rules: extractRuleGroup.rules.map(rule => ({
+                source: rule.source,
+                target: rule.target,
+                retainPathDepth: rule.retainPathDepth,
+                coerce: rule.coerce
+            }))
+        };
+    }
+    /**
+     * Remove a rule group.
+     * @param ruleGroupId The id of the rule group to remove.
+     * @returns A promise that resolves when the rule group has been removed.
+     */
+    async ruleGroupRemove(ruleGroupId) {
+        Guards.stringValue(DataProcessingService.CLASS_NAME, "ruleGroupId", ruleGroupId);
+        const extractRuleGroup = await this._extractionRuleGroupStorage.get(ruleGroupId);
+        if (Is.empty(extractRuleGroup)) {
+            throw new NotFoundError(DataProcessingService.CLASS_NAME, "ruleGroupNotFound", ruleGroupId);
+        }
+        await this._extractionRuleGroupStorage.remove(ruleGroupId);
+    }
+    /**
+     * Extracts data from the provided input.
+     * @param ruleGroupId The id of the rule group to use to extract data.
+     * @param data The data to extract from.
+     * @param overrideExtractorType An optional override for the extractor type.
+     * @param overrideMimeType An optional override for the mime type for conversion, will auto detect if not provided.
+     * @returns The extracted data.
+     */
+    async extract(ruleGroupId, data, overrideExtractorType, overrideMimeType) {
+        Guards.stringValue(DataProcessingService.CLASS_NAME, "ruleGroupId", ruleGroupId);
+        Guards.uint8Array(DataProcessingService.CLASS_NAME, "data", data);
+        const extractRuleGroup = await this._extractionRuleGroupStorage.get(ruleGroupId);
+        if (Is.empty(extractRuleGroup)) {
+            throw new NotFoundError(DataProcessingService.CLASS_NAME, "ruleGroupNotFound", ruleGroupId);
+        }
+        const converted = await this.convert(data, overrideMimeType);
+        const extractor = Is.empty(overrideExtractorType)
+            ? this._extractorConnector
+            : DataExtractorConnectorFactory.get(overrideExtractorType);
+        const extracted = await extractor.extract(converted, extractRuleGroup.rules);
+        return ObjectHelper.toExtended(extracted);
+    }
+    /**
+     * Converts data from the provided input to a structured JSON document.
+     * @param data The data to convert.
+     * @param overrideMimeType An optional override for the mime type, will auto detect if empty.
+     * @returns The converted data.
+     */
+    async convert(data, overrideMimeType) {
+        Guards.uint8Array(DataProcessingService.CLASS_NAME, "data", data);
+        const mimeType = overrideMimeType ?? (await MimeTypeHelper.detect(data));
+        if (Is.empty(mimeType)) {
+            throw new GeneralError(DataProcessingService.CLASS_NAME, "mimeTypeNotFound");
+        }
+        const converters = DataConverterConnectorFactory.instancesList();
+        const converter = converters.find(c => c.mimeTypes().includes(mimeType));
+        if (Is.empty(converter)) {
+            throw new NotFoundError(DataProcessingService.CLASS_NAME, "converterNotFound", mimeType);
+        }
+        return converter.convert(data);
+    }
+    /**
+     * Query the rule group entries.
+     * @param cursor The cursor to request the next chunk of entities.
+     * @param limit Limit the number of entities to return.
+     * @returns All the entities for the storage matching the conditions,
+     * and a cursor which can be used to request more entities.
+     */
+    async query(cursor, limit) {
+        const result = await this._extractionRuleGroupStorage.query(undefined, [
+            {
+                property: "id",
+                sortDirection: SortDirection.Descending
+            }
+        ], undefined, cursor, limit);
+        return {
+            entities: result.entities,
+            cursor: result.cursor
+        };
+    }
+}
+//# sourceMappingURL=dataProcessingService.js.map

package/dist/es/dataProcessingService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataProcessingService.js","sources":["../../src/dataProcessingService.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrF,OAAO,EACN,6BAA6B,EAC7B,6BAA6B,EAK7B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EACN,6BAA6B,EAE7B,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAI7C;;GAEG;AACH,MAAM,OAAO,qBAAqB;IACjC;;OAEG;IACI,MAAM,CAAU,UAAU,GAAW,uBAAuB,CAAC;IAEpE;;;OAGG;IACc,2BAA2B,CAA+C;IAE3F;;;OAGG;IACc,mBAAmB,CAA0B;IAE9D;;;;OAIG;IACH,YAAY,OAAkD;QAC7D,IAAI,CAAC,2BAA2B,GAAG,6BAA6B,CAAC,GAAG,CACnE,OAAO,EAAE,uCAAuC,IAAI,uBAAuB,CAC3E,CAAC;QAEF,IAAI,EAAE,CAAC,WAAW,CAAC,OAAO,EAAE,oBAAoB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,mBAAmB,GAAG,6BAA6B,CAAC,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC5F,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,6BAA6B,CAAC,KAAK,EAAE,CAAC;YACpD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,IAAI,CAAC,mBAAmB,GAAG,6BAA6B,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,CAAC;iBAAM,CAAC;gBACP,MAAM,IAAI,YAAY,CAAC,qBAAqB,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YAC9E,CAAC;QACF,CAAC;IACF,CAAC;IAED;;;OAGG;IACI,SAAS;QACf,OAAO,qBAAqB,CAAC,UAAU,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,YAAY,CAAC,SAAqB;QAC9C,MAAM,CAAC,MAAM,CAAa,qBAAqB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACpF,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,UAAU,EAAE,cAAc,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,UAAU,EAAE,iBAAiB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QACzF,MAAM,CAAC,KAAK,CAAQ,qBAAqB,CAAC,UAAU,EAAE,iBAAiB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QAE1F,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACpC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjF,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,mBAAmB,GAAwB;YAChD,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,MAAM,EAAE,IAAI,CAAC,MAAM;aACnB,CAAC,CAAC;SACH,CAAC;QAEF,MAAM,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjE,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,YAAY,CAAC,WAAmB;QAC5C,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,UAAU,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QAEjF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAEjF,IAAI,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,aAAa,CAAC,qBAAqB,CAAC,UAAU,EAAE,mBAAmB,EAAE,WAAW,CAAC,CAAC;QAC7F,CAAC;QAED,OAAO;YACN,EAAE,EAAE,gBAAgB,CAAC,EAAE;YACvB,KAAK,EAAE,gBAAgB,CAAC,KAAK;YAC7B,KAAK,EAAE,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC1C,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,MAAM,EAAE,IAAI,CAAC,MAAM;aACnB,CAAC,CAAC;SACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,eAAe,CAAC,WAAmB;QAC/C,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,UAAU,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QAEjF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAEjF,IAAI,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,aAAa,CAAC,qBAAqB,CAAC,UAAU,EAAE,mBAAmB,EAAE,WAAW,CAAC,CAAC;QAC7F,CAAC;QAED,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,OAAO,CACnB,WAAmB,EACnB,IAAgB,EAChB,qBAA8B,EAC9B,gBAAyB;QAEzB,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,UAAU,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QACjF,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAElE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAEjF,IAAI,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,aAAa,CAAC,qBAAqB,CAAC,UAAU,EAAE,mBAAmB,EAAE,WAAW,CAAC,CAAC;QAC7F,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAE7D,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC;YAChD,CAAC,CAAC,IAAI,CAAC,mBAAmB;YAC1B,CAAC,CAAC,6BAA6B,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAE5D,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC7E,OAAO,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,IAAgB,EAAE,gBAAyB;QAC/D,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAElE,MAAM,QAAQ,GAAG,gBAAgB,IAAI,CAAC,MAAM,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QACzE,IAAI,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,YAAY,CAAC,qBAAqB,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,UAAU,GAAG,6BAA6B,CAAC,aAAa,EAAE,CAAC;QAEjE,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACzE,IAAI,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,aAAa,CAAC,qBAAqB,CAAC,UAAU,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC;QAC1F,CAAC;QAED,OAAO,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,KAAK,CACjB,MAAe,EACf,KAAc;QAWd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAC1D,SAAS,EACT;YACC;gBACC,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,aAAa,CAAC,UAAU;aACvC;SACD,EACD,SAAS,EACT,MAAM,EACN,KAAK,CACL,CAAC;QAEF,OAAO;YACN,QAAQ,EAAE,MAAM,CAAC,QAAwB;YACzC,MAAM,EAAE,MAAM,CAAC,MAAM;SACrB,CAAC;IACH,CAAC;CACD","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport { GeneralError, Guards, Is, NotFoundError, ObjectHelper } from \"@3sixty/core\";\nimport {\n\tDataConverterConnectorFactory,\n\tDataExtractorConnectorFactory,\n\ttype IDataExtractorConnector,\n\ttype IDataProcessingComponent,\n\ttype IRule,\n\ttype IRuleGroup\n} from \"@3sixty/data-processing-models\";\nimport { SortDirection } from \"@3sixty/entity\";\nimport {\n\tEntityStorageConnectorFactory,\n\ttype IEntityStorageConnector\n} from \"@3sixty/entity-storage-models\";\n\nimport { MimeTypeHelper } from \"@3sixty/web\";\nimport type { ExtractionRuleGroup } from \"./entities/extractionRuleGroup.js\";\nimport type { IDataProcessingServiceConstructorOptions } from \"./models/IDataProcessingServiceConstructorOptions.js\";\n\n/**\n * Class for processing data from a source.\n */\nexport class DataProcessingService implements IDataProcessingComponent {\n\t/**\n\t * Runtime name for the class.\n\t */\n\tpublic static readonly CLASS_NAME: string = \"DataProcessingService\";\n\n\t/**\n\t * The entity storage for the extraction rule groups.\n\t * @internal\n\t */\n\tprivate readonly _extractionRuleGroupStorage: IEntityStorageConnector<ExtractionRuleGroup>;\n\n\t/**\n\t * The extractor connection to use.\n\t * @internal\n\t */\n\tprivate readonly _extractorConnector: IDataExtractorConnector;\n\n\t/**\n\t * Create a new instance of DataExtractionService.\n\t * @param options The options for the connector.\n\t * @throws GeneralError If no extractor connector is registered.\n\t */\n\tconstructor(options?: IDataProcessingServiceConstructorOptions) {\n\t\tthis._extractionRuleGroupStorage = EntityStorageConnectorFactory.get(\n\t\t\toptions?.extractionRuleGroupStorageConnectorType ?? \"extraction-rule-group\"\n\t\t);\n\n\t\tif (Is.stringValue(options?.defaultExtractorType)) {\n\t\t\tthis._extractorConnector = DataExtractorConnectorFactory.get(options.defaultExtractorType);\n\t\t} else {\n\t\t\tconst names = DataExtractorConnectorFactory.names();\n\t\t\tif (names.length > 0) {\n\t\t\t\tthis._extractorConnector = DataExtractorConnectorFactory.get(names[0]);\n\t\t\t} else {\n\t\t\t\tthrow new GeneralError(DataProcessingService.CLASS_NAME, \"noExtractorFound\");\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Returns the class name of the component.\n\t * @returns The class name of the component.\n\t */\n\tpublic className(): string {\n\t\treturn DataProcessingService.CLASS_NAME;\n\t}\n\n\t/**\n\t * Set an extraction rule group.\n\t * @param ruleGroup The rule group to store.\n\t * @returns A promise that resolves when the rule group has been stored.\n\t */\n\tpublic async ruleGroupSet(ruleGroup: IRuleGroup): Promise<void> {\n\t\tGuards.object<IRuleGroup>(DataProcessingService.CLASS_NAME, \"ruleGroup\", ruleGroup);\n\t\tGuards.stringValue(DataProcessingService.CLASS_NAME, \"ruleGroup.id\", ruleGroup.id);\n\t\tGuards.stringValue(DataProcessingService.CLASS_NAME, \"ruleGroup.label\", ruleGroup.label);\n\t\tGuards.array<IRule>(DataProcessingService.CLASS_NAME, \"ruleGroup.rules\", ruleGroup.rules);\n\n\t\tfor (const rule of ruleGroup.rules) {\n\t\t\tGuards.stringValue(DataProcessingService.CLASS_NAME, \"rule.source\", rule.source);\n\t\t\tGuards.stringValue(DataProcessingService.CLASS_NAME, \"rule.target\", rule.source);\n\t\t}\n\n\t\tconst extractionRuleGroup: ExtractionRuleGroup = {\n\t\t\tid: ruleGroup.id,\n\t\t\tlabel: ruleGroup.label,\n\t\t\trules: ruleGroup.rules.map(rule => ({\n\t\t\t\tsource: rule.source,\n\t\t\t\ttarget: rule.target,\n\t\t\t\tretainPathDepth: rule.retainPathDepth,\n\t\t\t\tcoerce: rule.coerce\n\t\t\t}))\n\t\t};\n\n\t\tawait this._extractionRuleGroupStorage.set(extractionRuleGroup);\n\t}\n\n\t/**\n\t * Get a rule group for extraction.\n\t * @param ruleGroupId The id of the rule group to get.\n\t * @returns The rule group.\n\t */\n\tpublic async ruleGroupGet(ruleGroupId: string): Promise<IRuleGroup> {\n\t\tGuards.stringValue(DataProcessingService.CLASS_NAME, \"ruleGroupId\", ruleGroupId);\n\n\t\tconst extractRuleGroup = await this._extractionRuleGroupStorage.get(ruleGroupId);\n\n\t\tif (Is.empty(extractRuleGroup)) {\n\t\t\tthrow new NotFoundError(DataProcessingService.CLASS_NAME, \"ruleGroupNotFound\", ruleGroupId);\n\t\t}\n\n\t\treturn {\n\t\t\tid: extractRuleGroup.id,\n\t\t\tlabel: extractRuleGroup.label,\n\t\t\trules: extractRuleGroup.rules.map(rule => ({\n\t\t\t\tsource: rule.source,\n\t\t\t\ttarget: rule.target,\n\t\t\t\tretainPathDepth: rule.retainPathDepth,\n\t\t\t\tcoerce: rule.coerce\n\t\t\t}))\n\t\t};\n\t}\n\n\t/**\n\t * Remove a rule group.\n\t * @param ruleGroupId The id of the rule group to remove.\n\t * @returns A promise that resolves when the rule group has been removed.\n\t */\n\tpublic async ruleGroupRemove(ruleGroupId: string): Promise<void> {\n\t\tGuards.stringValue(DataProcessingService.CLASS_NAME, \"ruleGroupId\", ruleGroupId);\n\n\t\tconst extractRuleGroup = await this._extractionRuleGroupStorage.get(ruleGroupId);\n\n\t\tif (Is.empty(extractRuleGroup)) {\n\t\t\tthrow new NotFoundError(DataProcessingService.CLASS_NAME, \"ruleGroupNotFound\", ruleGroupId);\n\t\t}\n\n\t\tawait this._extractionRuleGroupStorage.remove(ruleGroupId);\n\t}\n\n\t/**\n\t * Extracts data from the provided input.\n\t * @param ruleGroupId The id of the rule group to use to extract data.\n\t * @param data The data to extract from.\n\t * @param overrideExtractorType An optional override for the extractor type.\n\t * @param overrideMimeType An optional override for the mime type for conversion, will auto detect if not provided.\n\t * @returns The extracted data.\n\t */\n\tpublic async extract(\n\t\truleGroupId: string,\n\t\tdata: Uint8Array,\n\t\toverrideExtractorType?: string,\n\t\toverrideMimeType?: string\n\t): Promise<unknown> {\n\t\tGuards.stringValue(DataProcessingService.CLASS_NAME, \"ruleGroupId\", ruleGroupId);\n\t\tGuards.uint8Array(DataProcessingService.CLASS_NAME, \"data\", data);\n\n\t\tconst extractRuleGroup = await this._extractionRuleGroupStorage.get(ruleGroupId);\n\n\t\tif (Is.empty(extractRuleGroup)) {\n\t\t\tthrow new NotFoundError(DataProcessingService.CLASS_NAME, \"ruleGroupNotFound\", ruleGroupId);\n\t\t}\n\n\t\tconst converted = await this.convert(data, overrideMimeType);\n\n\t\tconst extractor = Is.empty(overrideExtractorType)\n\t\t\t? this._extractorConnector\n\t\t\t: DataExtractorConnectorFactory.get(overrideExtractorType);\n\n\t\tconst extracted = await extractor.extract(converted, extractRuleGroup.rules);\n\t\treturn ObjectHelper.toExtended(extracted);\n\t}\n\n\t/**\n\t * Converts data from the provided input to a structured JSON document.\n\t * @param data The data to convert.\n\t * @param overrideMimeType An optional override for the mime type, will auto detect if empty.\n\t * @returns The converted data.\n\t */\n\tpublic async convert(data: Uint8Array, overrideMimeType?: string): Promise<unknown> {\n\t\tGuards.uint8Array(DataProcessingService.CLASS_NAME, \"data\", data);\n\n\t\tconst mimeType = overrideMimeType ?? (await MimeTypeHelper.detect(data));\n\t\tif (Is.empty(mimeType)) {\n\t\t\tthrow new GeneralError(DataProcessingService.CLASS_NAME, \"mimeTypeNotFound\");\n\t\t}\n\n\t\tconst converters = DataConverterConnectorFactory.instancesList();\n\n\t\tconst converter = converters.find(c => c.mimeTypes().includes(mimeType));\n\t\tif (Is.empty(converter)) {\n\t\t\tthrow new NotFoundError(DataProcessingService.CLASS_NAME, \"converterNotFound\", mimeType);\n\t\t}\n\n\t\treturn converter.convert(data);\n\t}\n\n\t/**\n\t * Query the rule group entries.\n\t * @param cursor The cursor to request the next chunk of entities.\n\t * @param limit Limit the number of entities to return.\n\t * @returns All the entities for the storage matching the conditions,\n\t * and a cursor which can be used to request more entities.\n\t */\n\tpublic async query(\n\t\tcursor?: string,\n\t\tlimit?: number\n\t): Promise<{\n\t\t/**\n\t\t * The entities, which can be partial if a limited keys list was provided.\n\t\t */\n\t\tentities: IRuleGroup[];\n\t\t/**\n\t\t * An optional cursor, when defined can be used to call find to get more entities.\n\t\t */\n\t\tcursor?: string;\n\t}> {\n\t\tconst result = await this._extractionRuleGroupStorage.query(\n\t\t\tundefined,\n\t\t\t[\n\t\t\t\t{\n\t\t\t\t\tproperty: \"id\",\n\t\t\t\t\tsortDirection: SortDirection.Descending\n\t\t\t\t}\n\t\t\t],\n\t\t\tundefined,\n\t\t\tcursor,\n\t\t\tlimit\n\t\t);\n\n\t\treturn {\n\t\t\tentities: result.entities as IRuleGroup[],\n\t\t\tcursor: result.cursor\n\t\t};\n\t}\n}\n"]}

package/dist/es/entities/extractionRule.js

@@ -0,0 +1,43 @@
+import { entity, property } from "@3sixty/entity";
+/**
+ * Class defining an extraction rule.
+ */
+let ExtractionRule = class ExtractionRule {
+    /**
+     * The JSONPath expression identifying the source field in the input document.
+     */
+    source;
+    /**
+     * The dotted path identifying where to store the extracted value in the output.
+     */
+    target;
+    /**
+     * The number of path segments from the source location to preserve in the target path.
+     */
+    retainPathDepth;
+    /**
+     * The type to coerce the extracted value to.
+     */
+    coerce;
+};
+__decorate([
+    property({ type: "string", maxLength: 2048 }),
+    __metadata("design:type", String)
+], ExtractionRule.prototype, "source", void 0);
+__decorate([
+    property({ type: "string", maxLength: 2048 }),
+    __metadata("design:type", String)
+], ExtractionRule.prototype, "target", void 0);
+__decorate([
+    property({ type: "number", optional: true }),
+    __metadata("design:type", Number)
+], ExtractionRule.prototype, "retainPathDepth", void 0);
+__decorate([
+    property({ type: "string", maxLength: 16, optional: true }),
+    __metadata("design:type", String)
+], ExtractionRule.prototype, "coerce", void 0);
+ExtractionRule = __decorate([
+    entity()
+], ExtractionRule);
+export { ExtractionRule };
+//# sourceMappingURL=extractionRule.js.map

package/dist/es/entities/extractionRule.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractionRule.js","sources":["../../../src/entities/extractionRule.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAElD;;GAEG;AAEI,IAAM,cAAc,GAApB,MAAM,cAAc;IAC1B;;OAEG;IAEI,MAAM,CAAU;IAEvB;;OAEG;IAEI,MAAM,CAAU;IAEvB;;OAEG;IAEI,eAAe,CAAU;IAEhC;;OAEG;IAEI,MAAM,CAAc;CAC3B,CAAA;AAnBO;IADN,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;;8CACvB;AAMhB;IADN,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;;8CACvB;AAMhB;IADN,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;uDACb;AAMzB;IADN,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;8CACjC;AAvBf,cAAc;IAD1B,MAAM,EAAE;GACI,cAAc,CAwB1B","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport type { CoerceType } from \"@3sixty/core\";\nimport { entity, property } from \"@3sixty/entity\";\n\n/**\n * Class defining an extraction rule.\n */\n@entity()\nexport class ExtractionRule {\n\t/**\n\t * The JSONPath expression identifying the source field in the input document.\n\t */\n\t@property({ type: \"string\", maxLength: 2048 })\n\tpublic source!: string;\n\n\t/**\n\t * The dotted path identifying where to store the extracted value in the output.\n\t */\n\t@property({ type: \"string\", maxLength: 2048 })\n\tpublic target!: string;\n\n\t/**\n\t * The number of path segments from the source location to preserve in the target path.\n\t */\n\t@property({ type: \"number\", optional: true })\n\tpublic retainPathDepth?: number;\n\n\t/**\n\t * The type to coerce the extracted value to.\n\t */\n\t@property({ type: \"string\", maxLength: 16, optional: true })\n\tpublic coerce?: CoerceType;\n}\n"]}

package/dist/es/entities/extractionRuleGroup.js

@@ -0,0 +1,37 @@
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+import { entity, property } from "@3sixty/entity";
+/**
+ * Class defining an extraction rule group.
+ */
+let ExtractionRuleGroup = class ExtractionRuleGroup {
+    /**
+     * The unique identifier for the rule group.
+     */
+    id;
+    /**
+     * The human-readable label for the rule group.
+     */
+    label;
+    /**
+     * The extraction rules that belong to this group.
+     */
+    rules;
+};
+__decorate([
+    property({ type: "string", isPrimary: true, maxLength: 255 }),
+    __metadata("design:type", String)
+], ExtractionRuleGroup.prototype, "id", void 0);
+__decorate([
+    property({ type: "string", maxLength: 256 }),
+    __metadata("design:type", String)
+], ExtractionRuleGroup.prototype, "label", void 0);
+__decorate([
+    property({ type: "array", itemTypeRef: "ExtractionRule", itemType: "object" }),
+    __metadata("design:type", Array)
+], ExtractionRuleGroup.prototype, "rules", void 0);
+ExtractionRuleGroup = __decorate([
+    entity()
+], ExtractionRuleGroup);
+export { ExtractionRuleGroup };
+//# sourceMappingURL=extractionRuleGroup.js.map

package/dist/es/entities/extractionRuleGroup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractionRuleGroup.js","sources":["../../../src/entities/extractionRuleGroup.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAGlD;;GAEG;AAEI,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;IAC/B;;OAEG;IAEI,EAAE,CAAU;IAEnB;;OAEG;IAEI,KAAK,CAAU;IAEtB;;OAEG;IAEI,KAAK,CAAoB;CAChC,CAAA;AAbO;IADN,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;;+CAC3C;AAMZ;IADN,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;;kDACvB;AAMf;IADN,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;;kDAC/C;AAjBpB,mBAAmB;IAD/B,MAAM,EAAE;GACI,mBAAmB,CAkB/B","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport { entity, property } from \"@3sixty/entity\";\nimport type { ExtractionRule } from \"./extractionRule.js\";\n\n/**\n * Class defining an extraction rule group.\n */\n@entity()\nexport class ExtractionRuleGroup {\n\t/**\n\t * The unique identifier for the rule group.\n\t */\n\t@property({ type: \"string\", isPrimary: true, maxLength: 255 })\n\tpublic id!: string;\n\n\t/**\n\t * The human-readable label for the rule group.\n\t */\n\t@property({ type: \"string\", maxLength: 256 })\n\tpublic label!: string;\n\n\t/**\n\t * The extraction rules that belong to this group.\n\t */\n\t@property({ type: \"array\", itemTypeRef: \"ExtractionRule\", itemType: \"object\" })\n\tpublic rules!: ExtractionRule[];\n}\n"]}

package/dist/es/index.js

@@ -0,0 +1,10 @@
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+export * from "./dataProcessingRoutes.js";
+export * from "./dataProcessingService.js";
+export * from "./entities/extractionRule.js";
+export * from "./entities/extractionRuleGroup.js";
+export * from "./models/IDataProcessingServiceConstructorOptions.js";
+export * from "./restEntryPoints.js";
+export * from "./schema.js";
+//# sourceMappingURL=index.js.map

package/dist/es/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mCAAmC,CAAC;AAClD,cAAc,sDAAsD,CAAC;AACrE,cAAc,sBAAsB,CAAC;AACrC,cAAc,aAAa,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nexport * from \"./dataProcessingRoutes.js\";\nexport * from \"./dataProcessingService.js\";\nexport * from \"./entities/extractionRule.js\";\nexport * from \"./entities/extractionRuleGroup.js\";\nexport * from \"./models/IDataProcessingServiceConstructorOptions.js\";\nexport * from \"./restEntryPoints.js\";\nexport * from \"./schema.js\";\n"]}

package/dist/es/models/IDataProcessingServiceConstructorOptions.js

@@ -0,0 +1,4 @@
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+export {};
+//# sourceMappingURL=IDataProcessingServiceConstructorOptions.js.map

package/dist/es/models/IDataProcessingServiceConstructorOptions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IDataProcessingServiceConstructorOptions.js","sources":["../../../src/models/IDataProcessingServiceConstructorOptions.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\n\n/**\n * The options for the data processing service constructor.\n */\nexport interface IDataProcessingServiceConstructorOptions {\n\t/**\n\t * The type of the entity storage connector to use for extraction rule groups.\n\t * @default extraction-rule-group\n\t */\n\textractionRuleGroupStorageConnectorType?: string;\n\n\t/**\n\t * The default extractor type to use, if not provided uses the first in the factory.\n\t */\n\tdefaultExtractorType?: string;\n}\n"]}

package/dist/es/restEntryPoints.js

@@ -0,0 +1,13 @@
+import { generateRestRoutesDataProcessing, tagsDataProcessing } from "./dataProcessingRoutes.js";
+/**
+ * The REST entry points for the data processing service.
+ */
+export const restEntryPoints = [
+    {
+        name: "data-processing",
+        defaultBaseRoute: "data-processing",
+        tags: tagsDataProcessing,
+        generateRoutes: generateRestRoutesDataProcessing
+    }
+];
+//# sourceMappingURL=restEntryPoints.js.map

package/dist/es/restEntryPoints.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"restEntryPoints.js","sources":["../../src/restEntryPoints.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAEjG;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAA2B;IACtD;QACC,IAAI,EAAE,iBAAiB;QACvB,gBAAgB,EAAE,iBAAiB;QACnC,IAAI,EAAE,kBAAkB;QACxB,cAAc,EAAE,gCAAgC;KAChD;CACD,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport type { IRestRouteEntryPoint } from \"@3sixty/api-models\";\nimport { generateRestRoutesDataProcessing, tagsDataProcessing } from \"./dataProcessingRoutes.js\";\n\n/**\n * The REST entry points for the data processing service.\n */\nexport const restEntryPoints: IRestRouteEntryPoint[] = [\n\t{\n\t\tname: \"data-processing\",\n\t\tdefaultBaseRoute: \"data-processing\",\n\t\ttags: tagsDataProcessing,\n\t\tgenerateRoutes: generateRestRoutesDataProcessing\n\t}\n];\n"]}

package/dist/es/schema.js

@@ -0,0 +1,13 @@
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+import { EntitySchemaFactory, EntitySchemaHelper } from "@3sixty/entity";
+import { ExtractionRule } from "./entities/extractionRule.js";
+import { ExtractionRuleGroup } from "./entities/extractionRuleGroup.js";
+/**
+ * Registers entity schemas for the data extraction connector entity storage.
+ */
+export function initSchema() {
+    EntitySchemaFactory.register("ExtractionRuleGroup", () => EntitySchemaHelper.getSchema(ExtractionRuleGroup));
+    EntitySchemaFactory.register("ExtractionRule", () => EntitySchemaHelper.getSchema(ExtractionRule));
+}
+//# sourceMappingURL=schema.js.map

package/dist/es/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sources":["../../src/schema.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAExE;;GAEG;AACH,MAAM,UAAU,UAAU;IACzB,mBAAmB,CAAC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE,CACxD,kBAAkB,CAAC,SAAS,CAAC,mBAAmB,CAAC,CACjD,CAAC;IACF,mBAAmB,CAAC,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE,CACnD,kBAAkB,CAAC,SAAS,CAAC,cAAc,CAAC,CAC5C,CAAC;AACH,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport { EntitySchemaFactory, EntitySchemaHelper } from \"@3sixty/entity\";\n\nimport { ExtractionRule } from \"./entities/extractionRule.js\";\nimport { ExtractionRuleGroup } from \"./entities/extractionRuleGroup.js\";\n\n/**\n * Registers entity schemas for the data extraction connector entity storage.\n */\nexport function initSchema(): void {\n\tEntitySchemaFactory.register(\"ExtractionRuleGroup\", () =>\n\t\tEntitySchemaHelper.getSchema(ExtractionRuleGroup)\n\t);\n\tEntitySchemaFactory.register(\"ExtractionRule\", () =>\n\t\tEntitySchemaHelper.getSchema(ExtractionRule)\n\t);\n}\n"]}

package/dist/types/dataProcessingRoutes.d.ts

@@ -0,0 +1,61 @@
+import { type IHttpRequestContext, type INoContentResponse, type IRestRoute, type ITag } from "@3sixty/api-models";
+import type { IDataProcessingConvertRequest, IDataProcessingConvertResponse, IDataProcessingExtractRequest, IDataProcessingExtractResponse, IDataProcessingRuleGroupGetRequest, IDataProcessingRuleGroupGetResponse, IDataProcessingRuleGroupListRequest, IDataProcessingRuleGroupListResponse, IDataProcessingRuleGroupRemoveRequest, IDataProcessingRuleGroupSetRequest } from "@3sixty/data-processing-models";
+/**
+ * The tag to associate with the routes.
+ */
+export declare const tagsDataProcessing: ITag[];
+/**
+ * The REST routes for data processing.
+ * @param baseRouteName Prefix to prepend to the paths.
+ * @param componentName The name of the component to use in the routes stored in the ComponentFactory.
+ * @returns The generated routes.
+ */
+export declare function generateRestRoutesDataProcessing(baseRouteName: string, componentName: string): IRestRoute[];
+/**
+ * Store an extraction rule group.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function dataProcessingSetRuleGroup(httpRequestContext: IHttpRequestContext, componentName: string, request: IDataProcessingRuleGroupSetRequest): Promise<INoContentResponse>;
+/**
+ * Get an extraction rule group.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function dataProcessingGetRuleGroup(httpRequestContext: IHttpRequestContext, componentName: string, request: IDataProcessingRuleGroupGetRequest): Promise<IDataProcessingRuleGroupGetResponse>;
+/**
+ * Remove an extraction rule group.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function dataProcessingRemoveRuleGroup(httpRequestContext: IHttpRequestContext, componentName: string, request: IDataProcessingRuleGroupRemoveRequest): Promise<INoContentResponse>;
+/**
+ * Extract data using the specified rule group.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function dataProcessingExtract(httpRequestContext: IHttpRequestContext, componentName: string, request: IDataProcessingExtractRequest): Promise<IDataProcessingExtractResponse>;
+/**
+ * Convert data using the specified rule group.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function dataProcessingConvert(httpRequestContext: IHttpRequestContext, componentName: string, request: IDataProcessingConvertRequest): Promise<IDataProcessingConvertResponse>;
+/**
+ * Get a list of the rule group entries.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function ruleGroupList(httpRequestContext: IHttpRequestContext, componentName: string, request: IDataProcessingRuleGroupListRequest): Promise<IDataProcessingRuleGroupListResponse>;

package/dist/types/dataProcessingService.d.ts

@@ -0,0 +1,73 @@
+import { type IDataProcessingComponent, type IRuleGroup } from "@3sixty/data-processing-models";
+import type { IDataProcessingServiceConstructorOptions } from "./models/IDataProcessingServiceConstructorOptions.js";
+/**
+ * Class for processing data from a source.
+ */
+export declare class DataProcessingService implements IDataProcessingComponent {
+    /**
+     * Runtime name for the class.
+     */
+    static readonly CLASS_NAME: string;
+    /**
+     * Create a new instance of DataExtractionService.
+     * @param options The options for the connector.
+     * @throws GeneralError If no extractor connector is registered.
+     */
+    constructor(options?: IDataProcessingServiceConstructorOptions);
+    /**
+     * Returns the class name of the component.
+     * @returns The class name of the component.
+     */
+    className(): string;
+    /**
+     * Set an extraction rule group.
+     * @param ruleGroup The rule group to store.
+     * @returns A promise that resolves when the rule group has been stored.
+     */
+    ruleGroupSet(ruleGroup: IRuleGroup): Promise<void>;
+    /**
+     * Get a rule group for extraction.
+     * @param ruleGroupId The id of the rule group to get.
+     * @returns The rule group.
+     */
+    ruleGroupGet(ruleGroupId: string): Promise<IRuleGroup>;
+    /**
+     * Remove a rule group.
+     * @param ruleGroupId The id of the rule group to remove.
+     * @returns A promise that resolves when the rule group has been removed.
+     */
+    ruleGroupRemove(ruleGroupId: string): Promise<void>;
+    /**
+     * Extracts data from the provided input.
+     * @param ruleGroupId The id of the rule group to use to extract data.
+     * @param data The data to extract from.
+     * @param overrideExtractorType An optional override for the extractor type.
+     * @param overrideMimeType An optional override for the mime type for conversion, will auto detect if not provided.
+     * @returns The extracted data.
+     */
+    extract(ruleGroupId: string, data: Uint8Array, overrideExtractorType?: string, overrideMimeType?: string): Promise<unknown>;
+    /**
+     * Converts data from the provided input to a structured JSON document.
+     * @param data The data to convert.
+     * @param overrideMimeType An optional override for the mime type, will auto detect if empty.
+     * @returns The converted data.
+     */
+    convert(data: Uint8Array, overrideMimeType?: string): Promise<unknown>;
+    /**
+     * Query the rule group entries.
+     * @param cursor The cursor to request the next chunk of entities.
+     * @param limit Limit the number of entities to return.
+     * @returns All the entities for the storage matching the conditions,
+     * and a cursor which can be used to request more entities.
+     */
+    query(cursor?: string, limit?: number): Promise<{
+        /**
+         * The entities, which can be partial if a limited keys list was provided.
+         */
+        entities: IRuleGroup[];
+        /**
+         * An optional cursor, when defined can be used to call find to get more entities.
+         */
+        cursor?: string;
+    }>;
+}

package/dist/types/entities/extractionRule.d.ts

@@ -0,0 +1,22 @@
+import type { CoerceType } from "@3sixty/core";
+/**
+ * Class defining an extraction rule.
+ */
+export declare class ExtractionRule {
+    /**
+     * The JSONPath expression identifying the source field in the input document.
+     */
+    source: string;
+    /**
+     * The dotted path identifying where to store the extracted value in the output.
+     */
+    target: string;
+    /**
+     * The number of path segments from the source location to preserve in the target path.
+     */
+    retainPathDepth?: number;
+    /**
+     * The type to coerce the extracted value to.
+     */
+    coerce?: CoerceType;
+}

package/dist/types/entities/extractionRuleGroup.d.ts

@@ -0,0 +1,18 @@
+import type { ExtractionRule } from "./extractionRule.js";
+/**
+ * Class defining an extraction rule group.
+ */
+export declare class ExtractionRuleGroup {
+    /**
+     * The unique identifier for the rule group.
+     */
+    id: string;
+    /**
+     * The human-readable label for the rule group.
+     */
+    label: string;
+    /**
+     * The extraction rules that belong to this group.
+     */
+    rules: ExtractionRule[];
+}

package/dist/types/index.d.ts

@@ -0,0 +1,7 @@
+export * from "./dataProcessingRoutes.js";
+export * from "./dataProcessingService.js";
+export * from "./entities/extractionRule.js";
+export * from "./entities/extractionRuleGroup.js";
+export * from "./models/IDataProcessingServiceConstructorOptions.js";
+export * from "./restEntryPoints.js";
+export * from "./schema.js";

package/dist/types/models/IDataProcessingServiceConstructorOptions.d.ts

@@ -0,0 +1,14 @@
+/**
+ * The options for the data processing service constructor.
+ */
+export interface IDataProcessingServiceConstructorOptions {
+    /**
+     * The type of the entity storage connector to use for extraction rule groups.
+     * @default extraction-rule-group
+     */
+    extractionRuleGroupStorageConnectorType?: string;
+    /**
+     * The default extractor type to use, if not provided uses the first in the factory.
+     */
+    defaultExtractorType?: string;
+}

package/dist/types/restEntryPoints.d.ts

@@ -0,0 +1,5 @@
+import type { IRestRouteEntryPoint } from "@3sixty/api-models";
+/**
+ * The REST entry points for the data processing service.
+ */
+export declare const restEntryPoints: IRestRouteEntryPoint[];

package/dist/types/schema.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Registers entity schemas for the data extraction connector entity storage.
+ */
+export declare function initSchema(): void;

package/docs/changelog.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## [0.20.0-next.1](https://github.com/3sixtyglobal/data-processing/compare/data-processing-service-v0.20.0-next.0...data-processing-service-v0.20.0-next.1) (2026-10-23)
+
+
+### Features
+
+* publish packages under the [@3sixty](https://github.com/3sixty) namespace ([e62a15f](https://github.com/3sixtyglobal/data-processing/commit/e62a15f0f619ee7161c1f881a10a2d7db48dbc4d))
+* updated ts compile ([ba5bfb7](https://github.com/3sixtyglobal/data-processing/commit/ba5bfb7ce9fa96b17952fe0074192e73ccb4b1d6))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @3sixty/data-processing-models bumped from 0.20.0-next.0 to 0.20.0-next.1
+  * devDependencies
+    * @3sixty/data-processing-converters bumped from 0.20.0-next.0 to 0.20.0-next.1
+    * @3sixty/data-processing-extractors bumped from 0.20.0-next.0 to 0.20.0-next.1