@webiny/api-elasticsearch-tasks 6.3.0-beta.1 → 6.3.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/package.json +26 -30
  2. package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js +9 -0
  3. package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map +1 -1
package/package.json CHANGED
@@ -1,8 +1,11 @@
1
1
  {
2
2
  "name": "@webiny/api-elasticsearch-tasks",
3
- "version": "6.3.0-beta.1",
3
+ "version": "6.3.0-beta.3",
4
4
  "type": "module",
5
- "main": "index.js",
5
+ "exports": {
6
+ ".": "./index.js",
7
+ "./*": "./*"
8
+ },
6
9
  "repository": {
7
10
  "type": "git",
8
11
  "url": "https://github.com/webiny/webiny-js.git"
@@ -13,43 +16,36 @@
13
16
  ],
14
17
  "license": "MIT",
15
18
  "dependencies": {
16
- "@webiny/api": "6.3.0-beta.1",
17
- "@webiny/api-core": "6.3.0-beta.1",
18
- "@webiny/api-dynamodb-to-elasticsearch": "6.3.0-beta.1",
19
- "@webiny/api-opensearch": "6.3.0-beta.1",
20
- "@webiny/aws-sdk": "6.3.0-beta.1",
21
- "@webiny/db": "6.3.0-beta.1",
22
- "@webiny/db-dynamodb": "6.3.0-beta.1",
23
- "@webiny/error": "6.3.0-beta.1",
24
- "@webiny/feature": "6.3.0-beta.1",
25
- "@webiny/plugins": "6.3.0-beta.1",
26
- "@webiny/tasks": "6.3.0-beta.1",
27
- "@webiny/utils": "6.3.0-beta.1"
19
+ "@webiny/api": "6.3.0-beta.3",
20
+ "@webiny/api-core": "6.3.0-beta.3",
21
+ "@webiny/api-dynamodb-to-elasticsearch": "6.3.0-beta.3",
22
+ "@webiny/api-opensearch": "6.3.0-beta.3",
23
+ "@webiny/aws-sdk": "6.3.0-beta.3",
24
+ "@webiny/db": "6.3.0-beta.3",
25
+ "@webiny/db-dynamodb": "6.3.0-beta.3",
26
+ "@webiny/error": "6.3.0-beta.3",
27
+ "@webiny/feature": "6.3.0-beta.3",
28
+ "@webiny/plugins": "6.3.0-beta.3",
29
+ "@webiny/tasks": "6.3.0-beta.3",
30
+ "@webiny/utils": "6.3.0-beta.3"
28
31
  },
29
32
  "devDependencies": {
30
33
  "@webiny/api": "0.0.0",
31
- "@webiny/api-headless-cms": "6.3.0-beta.1",
32
- "@webiny/build-tools": "6.3.0-beta.1",
33
- "@webiny/handler": "6.3.0-beta.1",
34
- "@webiny/handler-aws": "6.3.0-beta.1",
35
- "@webiny/handler-db": "6.3.0-beta.1",
36
- "@webiny/handler-graphql": "6.3.0-beta.1",
37
- "@webiny/project-utils": "6.3.0-beta.1",
34
+ "@webiny/api-headless-cms": "6.3.0-beta.3",
35
+ "@webiny/build-tools": "6.3.0-beta.3",
36
+ "@webiny/handler": "6.3.0-beta.3",
37
+ "@webiny/handler-aws": "6.3.0-beta.3",
38
+ "@webiny/handler-db": "6.3.0-beta.3",
39
+ "@webiny/handler-graphql": "6.3.0-beta.3",
40
+ "@webiny/project-utils": "6.3.0-beta.3",
38
41
  "rimraf": "6.1.3",
39
42
  "type-fest": "5.6.0",
40
43
  "typescript": "6.0.3",
41
- "vitest": "4.1.4"
44
+ "vitest": "4.1.5"
42
45
  },
43
46
  "publishConfig": {
44
47
  "access": "public",
45
48
  "directory": "dist"
46
49
  },
47
- "gitHead": "664b273a9f0a971f9ca7e6ffe920db77fefdced1",
48
- "adio": {
49
- "ignore": {
50
- "src": [
51
- "node:util"
52
- ]
53
- }
54
- }
50
+ "gitHead": "e154ec3326903876c357d35422dc60d29e061419"
55
51
  }
package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js CHANGED
@@ -61,13 +61,20 @@ export class ElasticsearchFetcher {
61
61
  items: []
62
62
  };
63
63
  }
64
+
65
+ /**
66
+ * TODO expect errors over hit properties is required due to opensearch library narrowing types too much because of the _source: false. At least what Claude says, didnt go into it too much.
67
+ * Properties are there, but types are not correct.
68
+ */
64
69
  const hasMoreItems = hits.length > limit;
65
70
  let nextCursor;
66
71
  if (hasMoreItems) {
67
72
  hits.pop();
73
+ // @ts-expect-error
68
74
  nextCursor = hits.at(-1)?.sort;
69
75
  }
70
76
  const items = hits.reduce((collection, hit) => {
77
+ // @ts-expect-error
71
78
  const [PK, SK] = hit._id.split(":");
72
79
  if (!PK || !SK) {
73
80
  return collection;
@@ -75,7 +82,9 @@ export class ElasticsearchFetcher {
75
82
  collection.push({
76
83
  PK,
77
84
  SK,
85
+ // @ts-expect-error
78
86
  _id: hit._id,
87
+ // @ts-expect-error
79
88
  index: hit._index
80
89
  });
81
90
  return collection;
package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"names":["getTotalCount","shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-opensearch\";\nimport type {\n IElasticsearchFetcher,\n IElasticsearchFetcherFetchParams,\n IElasticsearchFetcherFetchResponse,\n IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher.js\";\nimport {\n type OpenSearchSearchResponse,\n type PrimitiveValue,\n getTotalCount\n} from \"@webiny/api-opensearch/types.js\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError.js\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n private readonly client: Client;\n\n public constructor(params: IElasticsearchFetcherParams) {\n this.client = params.client;\n }\n public async fetch({\n index,\n cursor,\n limit\n }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n let response: OpenSearchSearchResponse;\n try {\n response = await this.client.search({\n index,\n body: {\n query: {\n match_all: {}\n },\n sort: {\n \"id.keyword\": {\n order: \"asc\"\n }\n },\n size: limit + 1,\n track_total_hits: true,\n search_after: cursor,\n _source: false\n }\n });\n } catch (ex) {\n /**\n * If we ignore the error, we can continue with the next index.\n */\n if (shouldIgnoreEsResponseError(ex)) {\n if (process.env.DEBUG === \"true\") {\n console.error(\n inspect(ex, {\n depth: 5,\n showHidden: true\n })\n );\n }\n return {\n done: true,\n totalCount: 0,\n items: []\n };\n }\n console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n throw ex;\n }\n\n const { hits, total } = response.body.hits;\n if (hits.length === 0) {\n return {\n done: true,\n cursor: undefined,\n totalCount: getTotalCount(total),\n items: []\n };\n }\n\n const hasMoreItems = hits.length > limit;\n let nextCursor: PrimitiveValue[] | undefined;\n if (hasMoreItems) {\n hits.pop();\n nextCursor = hits.at(-1)?.sort;\n }\n const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n const [PK, SK] = hit._id.split(\":\");\n if (!PK || !SK) {\n return collection;\n }\n collection.push({\n PK,\n SK,\n _id: hit._id,\n index: hit._index\n });\n\n return collection;\n }, []);\n\n return {\n totalCount: getTotalCount(total),\n cursor: nextCursor,\n done: !nextCursor,\n items\n };\n }\n}\n"],"mappings":"AAOA,SAGIA,aAAa,QACV,iCAAiC;AACxC,SAASC,2BAA2B;AACpC,SAASC,OAAO,QAAQ,WAAW;AAMnC,OAAO,MAAMC,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAAkC;IACtC,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAIrB,2BAA2B,CAACqB,EAAE,CAAC,EAAE;QACjC,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACTzB,OAAO,CAACoB,EAAE,EAAE;YACRM,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAN,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEL,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEW,IAAI;MAAEC;IAAM,CAAC,GAAGvB,QAAQ,CAACE,IAAI,CAACoB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVrB,MAAM,EAAE2B,SAAS;QACjBL,UAAU,EAAE/B,aAAa,CAACkC,KAAK,CAAC;QAChCF,KAAK,EAAE;MACX,CAAC;IACL;IAEA,MAAMK,YAAY,GAAGJ,IAAI,CAACE,MAAM,GAAGzB,KAAK;IACxC,IAAI4B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdJ,IAAI,CAACM,GAAG,CAAC,CAAC;MACVD,UAAU,GAAGL,IAAI,CAACO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAExB,IAAI;IAClC;IACA,MAAMgB,KAAK,GAAGC,IAAI,CAACQ,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACFC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZtC,KAAK,EAAEmC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHX,UAAU,EAAE/B,aAAa,CAACkC,KAAK,CAAC;MAChCzB,MAAM,EAAE6B,UAAU;MAClBR,IAAI,EAAE,CAACQ,UAAU;MACjBN;IACJ,CAAC;EACL;AACJ","ignoreList":[]}
1
+ {"version":3,"names":["getTotalCount","shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-opensearch\";\nimport type {\n IElasticsearchFetcher,\n IElasticsearchFetcherFetchParams,\n IElasticsearchFetcherFetchResponse,\n IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher.js\";\nimport {\n type OpenSearchSearchResponse,\n type PrimitiveValue,\n getTotalCount\n} from \"@webiny/api-opensearch/types.js\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError.js\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n private readonly client: Client;\n\n public constructor(params: IElasticsearchFetcherParams) {\n this.client = params.client;\n }\n public async fetch({\n index,\n cursor,\n limit\n }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n let response: OpenSearchSearchResponse;\n try {\n response = await this.client.search({\n index,\n body: {\n query: {\n match_all: {}\n },\n sort: {\n \"id.keyword\": {\n order: \"asc\"\n }\n },\n size: limit + 1,\n track_total_hits: true,\n search_after: cursor,\n _source: false\n }\n });\n } catch (ex) {\n /**\n * If we ignore the error, we can continue with the next index.\n */\n if (shouldIgnoreEsResponseError(ex)) {\n if (process.env.DEBUG === \"true\") {\n console.error(\n inspect(ex, {\n depth: 5,\n showHidden: true\n })\n );\n }\n return {\n done: true,\n totalCount: 0,\n items: []\n };\n }\n console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n throw ex;\n }\n\n const { hits, total } = response.body.hits;\n if (hits.length === 0) {\n return {\n done: true,\n cursor: undefined,\n totalCount: getTotalCount(total),\n items: []\n };\n }\n\n /**\n * TODO expect errors over hit properties is required due to opensearch library narrowing types too much because of the _source: false. At least what Claude says, didnt go into it too much.\n * Properties are there, but types are not correct.\n */\n const hasMoreItems = hits.length > limit;\n let nextCursor: PrimitiveValue[] | undefined;\n if (hasMoreItems) {\n hits.pop();\n // @ts-expect-error\n nextCursor = hits.at(-1)?.sort;\n }\n const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n // @ts-expect-error\n const [PK, SK] = hit._id.split(\":\");\n if (!PK || !SK) {\n return collection;\n }\n collection.push({\n PK,\n SK,\n // @ts-expect-error\n _id: hit._id,\n // @ts-expect-error\n index: hit._index\n });\n\n return collection;\n }, []);\n\n return {\n totalCount: getTotalCount(total),\n cursor: nextCursor,\n done: !nextCursor,\n items\n };\n }\n}\n"],"mappings":"AAOA,SAGIA,aAAa,QACV,iCAAiC;AACxC,SAASC,2BAA2B;AACpC,SAASC,OAAO,QAAQ,WAAW;AAMnC,OAAO,MAAMC,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAAkC;IACtC,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAIrB,2BAA2B,CAACqB,EAAE,CAAC,EAAE;QACjC,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACTzB,OAAO,CAACoB,EAAE,EAAE;YACRM,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAN,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEL,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEW,IAAI;MAAEC;IAAM,CAAC,GAAGvB,QAAQ,CAACE,IAAI,CAACoB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVrB,MAAM,EAAE2B,SAAS;QACjBL,UAAU,EAAE/B,aAAa,CAACkC,KAAK,CAAC;QAChCF,KAAK,EAAE;MACX,CAAC;IACL;;IAEA;AACR;AACA;AACA;IACQ,MAAMK,YAAY,GAAGJ,IAAI,CAACE,MAAM,GAAGzB,KAAK;IACxC,IAAI4B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdJ,IAAI,CAACM,GAAG,CAAC,CAAC;MACV;MACAD,UAAU,GAAGL,IAAI,CAACO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAExB,IAAI;IAClC;IACA,MAAMgB,KAAK,GAAGC,IAAI,CAACQ,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF;MACA,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACF;QACAC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZ;QACAtC,KAAK,EAAEmC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHX,UAAU,EAAE/B,aAAa,CAACkC,KAAK,CAAC;MAChCzB,MAAM,EAAE6B,UAAU;MAClBR,IAAI,EAAE,CAACQ,UAAU;MACjBN;IACJ,CAAC;EACL;AACJ","ignoreList":[]}