@twin.org/data-processing-extractors 0.0.3-next.2 → 0.9.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -41,6 +41,7 @@ export class JsonPathExtractorConnector {
|
|
|
41
41
|
* @param jsonObject The JSON object to extract from.
|
|
42
42
|
* @param rule The rule to use to extract the data.
|
|
43
43
|
* @param outputObject The object to output the extracted data to.
|
|
44
|
+
* @throws GeneralError if the rule is invalid or extraction fails.
|
|
44
45
|
* @internal
|
|
45
46
|
*/
|
|
46
47
|
extractValue(jsonObject, rule, outputObject) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonPathExtractorConnector.js","sourceRoot":"","sources":["../../../src/extractors/jsonPathExtractorConnector.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3F,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAI1D;;GAEG;AACH,MAAM,OAAO,0BAA0B;IACtC;;OAEG;IACI,MAAM,CAAU,SAAS,GAAW,WAAW,CAAC;IAEvD;;OAEG;IACI,MAAM,CAAU,UAAU,gCAAgD;IAEjF;;;OAGG;IACI,SAAS;QACf,OAAO,0BAA0B,CAAC,UAAU,CAAC;IAC9C,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,IAAa,EAAE,KAAc;QACjD,MAAM,CAAC,MAAM,CAAC,0BAA0B,CAAC,UAAU,UAAgB,IAAI,CAAC,CAAC;QACzE,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,UAAU,WAAiB,KAAK,CAAC,CAAC;QAE1E,MAAM,YAAY,GAAY,EAAE,CAAC;QAEjC,KAAK,MAAM,WAAW,IAAI,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,YAAY,CAAC;IACrB,CAAC;IAED
|
|
1
|
+
{"version":3,"file":"jsonPathExtractorConnector.js","sourceRoot":"","sources":["../../../src/extractors/jsonPathExtractorConnector.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3F,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAI1D;;GAEG;AACH,MAAM,OAAO,0BAA0B;IACtC;;OAEG;IACI,MAAM,CAAU,SAAS,GAAW,WAAW,CAAC;IAEvD;;OAEG;IACI,MAAM,CAAU,UAAU,gCAAgD;IAEjF;;;OAGG;IACI,SAAS;QACf,OAAO,0BAA0B,CAAC,UAAU,CAAC;IAC9C,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,IAAa,EAAE,KAAc;QACjD,MAAM,CAAC,MAAM,CAAC,0BAA0B,CAAC,UAAU,UAAgB,IAAI,CAAC,CAAC;QACzE,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,UAAU,WAAiB,KAAK,CAAC,CAAC;QAE1E,MAAM,YAAY,GAAY,EAAE,CAAC;QAEjC,KAAK,MAAM,WAAW,IAAI,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,YAAY,CAAC;IACrB,CAAC;IAED;;;;;;;OAOG;IACK,YAAY,CAAC,UAAmB,EAAE,IAAW,EAAE,YAAqB;QAC3E,IAAI,CAAC;YACJ,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAEhE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAEzC,yEAAyE;YACzE,MAAM,YAAY,GAA8B,EAAE,CAAC;YAEnD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBAClC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;gBAClD,MAAM,cAAc,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC/H,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEhE,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC;gBAEvC,4EAA4E;gBAC5E,8EAA8E;gBAC9E,MAAM,OAAO,GACZ,YAAY,CAAC,MAAM,GAAG,CAAC;oBACvB,SAAS,CAAC,MAAM,GAAG,CAAC;oBACpB,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACjD,SAAS,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAEpD,IAAI,OAAO,EAAE,CAAC;oBACb,oEAAoE;oBACpE,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;wBACxF,YAAY,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,CAAC;oBAC/D,CAAC;oBAED,6BAA6B;oBAC7B,YAAY,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;oBACpC,wBAAwB;oBACxB,IAAI,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;wBAC5C,YAAY,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACjD,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,iDAAiD;oBACjD,YAAY,CAAC,cAAc,CAAC,GAAG,YAAY,CAAC;gBAC7C,CAAC;YACF,CAAC;YAED,sCAAsC;YACtC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC3D,YAAY,CAAC,WAAW,CAAC,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YACtD,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,YAAY,CACrB,0BAA0B,CAAC,UAAU,EACrC,aAAa,EACb;gBACC,IAAI,EAAE,IAAI,CAAC,MAAM;aACjB,EACD,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CACxB,CAAC;QACH,CAAC;IACF,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport { BaseError, Coerce, GeneralError, Guards, Is, ObjectHelper } from \"@twin.org/core\";\nimport { JsonPathHelper } from \"@twin.org/data-json-path\";\nimport type { IDataExtractorConnector, IRule } from \"@twin.org/data-processing-models\";\nimport { nameof } from \"@twin.org/nameof\";\n\n/**\n * Class for extracting data from a JSON source.\n */\nexport class JsonPathExtractorConnector implements IDataExtractorConnector {\n\t/**\n\t * The namespace supported by the data extractor connector.\n\t */\n\tpublic static readonly NAMESPACE: string = \"json-path\";\n\n\t/**\n\t * Runtime name for the class.\n\t */\n\tpublic static readonly CLASS_NAME: string = nameof<JsonPathExtractorConnector>();\n\n\t/**\n\t * Returns the class name of the component.\n\t * @returns The class name of the component.\n\t */\n\tpublic className(): string {\n\t\treturn JsonPathExtractorConnector.CLASS_NAME;\n\t}\n\n\t/**\n\t * Extracts data from the provided input.\n\t * @param data The object to extract from.\n\t * @param rules The rules to use to extract the data.\n\t * @returns The extracted data.\n\t */\n\tpublic async extract(data: unknown, rules: IRule[]): Promise<unknown> {\n\t\tGuards.object(JsonPathExtractorConnector.CLASS_NAME, nameof(data), data);\n\t\tGuards.array(JsonPathExtractorConnector.CLASS_NAME, nameof(rules), rules);\n\n\t\tconst outputObject: unknown = {};\n\n\t\tfor (const extractRule of rules) {\n\t\t\tthis.extractValue(data, extractRule, outputObject);\n\t\t}\n\n\t\treturn outputObject;\n\t}\n\n\t/**\n\t * Extracts the value from the JSON object.\n\t * @param jsonObject The JSON object to extract from.\n\t * @param rule The rule to use to extract the data.\n\t * @param outputObject The object to output the extracted data to.\n\t * @throws GeneralError if the rule is invalid or extraction fails.\n\t * @internal\n\t */\n\tprivate extractValue(jsonObject: unknown, rule: IRule, outputObject: unknown): void {\n\t\ttry {\n\t\t\tconst jsonNodes = JsonPathHelper.query(rule.source, jsonObject);\n\n\t\t\tconst ruleParts = rule.source.split(\".\");\n\n\t\t\t// Group values by their target path to handle multiple results correctly\n\t\t\tconst pathToValues: { [id: string]: unknown } = {};\n\n\t\t\tfor (const jsonNode of jsonNodes) {\n\t\t\t\tconst retainPathDepth = rule.retainPathDepth ?? 0;\n\t\t\t\tconst fullTargetPath = `${rule.target}${retainPathDepth > 0 ? `.${jsonNode.location.slice(-retainPathDepth).join(\".\")}` : \"\"}`;\n\t\t\t\tconst coercedValue = Coerce.byType(jsonNode.value, rule.coerce);\n\n\t\t\t\tconst nodeLocation = jsonNode.location;\n\n\t\t\t\t// If the last part of the location before the target property is an integer\n\t\t\t\t// and the corresponding rule part ends with [*], we are dealing with an array\n\t\t\t\tconst isArray =\n\t\t\t\t\tnodeLocation.length > 1 &&\n\t\t\t\t\truleParts.length > 1 &&\n\t\t\t\t\tIs.integer(nodeLocation[nodeLocation.length - 2]) &&\n\t\t\t\t\truleParts[nodeLocation.length - 2].endsWith(\"[*]\");\n\n\t\t\t\tif (isArray) {\n\t\t\t\t\t// If the value is set and not currently an array, convert it to one\n\t\t\t\t\tif (!Is.empty(pathToValues[fullTargetPath]) && !Is.array(pathToValues[fullTargetPath])) {\n\t\t\t\t\t\tpathToValues[fullTargetPath] = [pathToValues[fullTargetPath]];\n\t\t\t\t\t}\n\n\t\t\t\t\t// Make sure the array exists\n\t\t\t\t\tpathToValues[fullTargetPath] ??= [];\n\t\t\t\t\t// And add the new value\n\t\t\t\t\tif (Is.array(pathToValues[fullTargetPath])) {\n\t\t\t\t\t\tpathToValues[fullTargetPath].push(coercedValue);\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// For single values, just set the value directly\n\t\t\t\t\tpathToValues[fullTargetPath] = coercedValue;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Set the values in the output object\n\t\t\tfor (const [path, values] of Object.entries(pathToValues)) {\n\t\t\t\tObjectHelper.propertySet(outputObject, path, values);\n\t\t\t}\n\t\t} catch (err) {\n\t\t\tthrow new GeneralError(\n\t\t\t\tJsonPathExtractorConnector.CLASS_NAME,\n\t\t\t\t\"invalidRule\",\n\t\t\t\t{\n\t\t\t\t\trule: rule.source\n\t\t\t\t},\n\t\t\t\tBaseError.fromError(err)\n\t\t\t);\n\t\t}\n\t}\n}\n"]}
|
package/docs/changelog.md
CHANGED
|
@@ -1,5 +1,46 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.9.0-next.1](https://github.com/iotaledger/twin-data-processing/compare/data-processing-extractors-v0.9.0-next.0...data-processing-extractors-v0.9.0-next.1) (2026-06-23)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* add context id features ([#25](https://github.com/iotaledger/twin-data-processing/issues/25)) ([2a2a955](https://github.com/iotaledger/twin-data-processing/commit/2a2a9559d54e8c703e97e2c05996e678416ef560))
|
|
9
|
+
* Add rule group query functionality ([#2](https://github.com/iotaledger/twin-data-processing/issues/2)) ([0fbbfb0](https://github.com/iotaledger/twin-data-processing/commit/0fbbfb065b6ecc293920b25f97ba011743105486))
|
|
10
|
+
* add validate-locales ([9af0874](https://github.com/iotaledger/twin-data-processing/commit/9af0874cd3f44a4cfca514dde8380f002e62337e))
|
|
11
|
+
* eslint migration to flat config ([a8138b0](https://github.com/iotaledger/twin-data-processing/commit/a8138b020ed58baeff162a822d4df207a6aeadc4))
|
|
12
|
+
* typescript 6 update ([b67debb](https://github.com/iotaledger/twin-data-processing/commit/b67debb8aff83c0f3b179c6db038ed733f7308b1))
|
|
13
|
+
* update dependencies ([0b2e77d](https://github.com/iotaledger/twin-data-processing/commit/0b2e77d673ede6e92847ff0b50fdfcf4e9c08b3b))
|
|
14
|
+
* update dependencies ([21b8e80](https://github.com/iotaledger/twin-data-processing/commit/21b8e8007c87136a09f0a8e35ffde13a07ff4711))
|
|
15
|
+
* update framework core ([96f6e80](https://github.com/iotaledger/twin-data-processing/commit/96f6e8016a4ef0fb873f6775e750b4ab42e9f491))
|
|
16
|
+
* use shared store mechanism ([#6](https://github.com/iotaledger/twin-data-processing/issues/6)) ([6009228](https://github.com/iotaledger/twin-data-processing/commit/600922880acef07cc2f818dee7645c342929108b))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
### Bug Fixes
|
|
20
|
+
|
|
21
|
+
* supporting array usage ([#20](https://github.com/iotaledger/twin-data-processing/issues/20)) ([d2c68cd](https://github.com/iotaledger/twin-data-processing/commit/d2c68cd3983622586cc073377545d6416b87a827))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
### Dependencies
|
|
25
|
+
|
|
26
|
+
* The following workspace dependencies were updated
|
|
27
|
+
* dependencies
|
|
28
|
+
* @twin.org/data-processing-models bumped from 0.9.0-next.0 to 0.9.0-next.1
|
|
29
|
+
|
|
30
|
+
## [0.0.3-next.3](https://github.com/iotaledger/twin-data-processing/compare/data-processing-extractors-v0.0.3-next.2...data-processing-extractors-v0.0.3-next.3) (2026-05-20)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
### Features
|
|
34
|
+
|
|
35
|
+
* update dependencies ([0b2e77d](https://github.com/iotaledger/twin-data-processing/commit/0b2e77d673ede6e92847ff0b50fdfcf4e9c08b3b))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
### Dependencies
|
|
39
|
+
|
|
40
|
+
* The following workspace dependencies were updated
|
|
41
|
+
* dependencies
|
|
42
|
+
* @twin.org/data-processing-models bumped from 0.0.3-next.2 to 0.0.3-next.3
|
|
43
|
+
|
|
3
44
|
## [0.0.3-next.2](https://github.com/iotaledger/twin-data-processing/compare/data-processing-extractors-v0.0.3-next.1...data-processing-extractors-v0.0.3-next.2) (2026-05-11)
|
|
4
45
|
|
|
5
46
|
|
package/package.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@twin.org/data-processing-extractors",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.9.0-next.1",
|
|
4
4
|
"description": "Connector implementations that extract data from external systems for processing",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
|
-
"url": "git+https://github.com/iotaledger/data-processing.git",
|
|
7
|
+
"url": "git+https://github.com/iotaledger/twin-data-processing.git",
|
|
8
8
|
"directory": "packages/data-processing-extractors"
|
|
9
9
|
},
|
|
10
10
|
"author": "martyn.janes@iota.org",
|
|
@@ -14,11 +14,11 @@
|
|
|
14
14
|
"node": ">=20.0.0"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@twin.org/core": "next",
|
|
18
|
-
"@twin.org/data-json-path": "next",
|
|
19
|
-
"@twin.org/data-processing-models": "0.0
|
|
20
|
-
"@twin.org/nameof": "next",
|
|
21
|
-
"@twin.org/web": "next"
|
|
17
|
+
"@twin.org/core": "0.9.0-next.1",
|
|
18
|
+
"@twin.org/data-json-path": "0.9.0-next.1",
|
|
19
|
+
"@twin.org/data-processing-models": "0.9.0-next.1",
|
|
20
|
+
"@twin.org/nameof": "0.9.0-next.1",
|
|
21
|
+
"@twin.org/web": "0.9.0-next.1"
|
|
22
22
|
},
|
|
23
23
|
"main": "./dist/es/index.js",
|
|
24
24
|
"types": "./dist/types/index.d.ts",
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"conversion"
|
|
50
50
|
],
|
|
51
51
|
"bugs": {
|
|
52
|
-
"url": "git+https://github.com/iotaledger/data-processing/issues"
|
|
52
|
+
"url": "git+https://github.com/iotaledger/twin-data-processing/issues"
|
|
53
53
|
},
|
|
54
54
|
"homepage": "https://twindev.org"
|
|
55
55
|
}
|