@owox/connectors 0.35.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/dist/connector-runner.cjs +61 -25
  2. package/dist/connector-runner.js +61 -25
  3. package/dist/index.cjs +9553 -6218
  4. package/dist/index.js +36408 -33082
  5. package/package.json +12 -3
  6. package/types/index.d.ts +36 -0
package/dist/connector-runner.cjs CHANGED
@@ -64,6 +64,7 @@ Object.keys(Connectors).forEach((key) => {
64
64
  global[key2] = connector[key2];
65
65
  });
66
66
  });
67
+ let runContext = null;
67
68
  async function main() {
68
69
  if (!process.env.OW_CONFIG) {
69
70
  throw new Error("OW_CONFIG environment variable is required");
@@ -74,40 +75,75 @@ async function main() {
74
75
  if (!process.env.OW_RUN_ID) {
75
76
  throw new Error("OW_RUN_ID environment variable is required");
76
77
  }
78
+ const isTestMode = Boolean(process.env.OW_TEST);
79
+ const testLimits = isTestMode ? {
80
+ maxRows: process.env.OW_TEST_MAX_ROWS ? parseInt(process.env.OW_TEST_MAX_ROWS, 10) : void 0,
81
+ maxPages: process.env.OW_TEST_MAX_PAGES ? parseInt(process.env.OW_TEST_MAX_PAGES, 10) : void 0,
82
+ maxSlices: process.env.OW_TEST_MAX_SLICES ? parseInt(process.env.OW_TEST_MAX_SLICES, 10) : 10
83
+ } : {};
77
84
  let envConfig;
78
85
  try {
79
86
  envConfig = JSON.parse(process.env.OW_CONFIG);
80
87
  } catch (error) {
81
88
  throw new Error(`Failed to parse OW_CONFIG: ${error.message}`);
82
89
  }
83
- const config = new Core.NodeJsConfig(envConfig);
84
- const runConfigJson = process.env.OW_RUN_CONFIG;
85
- const runConfig = runConfigJson ? new Core.AbstractRunConfig(JSON.parse(runConfigJson)) : new Core.AbstractRunConfig();
86
- const sourceName = config.getSourceName();
87
- const storageName = config.getStorageName();
88
- const sourceClass = global[sourceName];
89
- if (!sourceClass) {
90
- throw new Error(`Source class ${sourceName} not found`);
90
+ let runConfigData = null;
91
+ if (process.env.OW_RUN_CONFIG) {
92
+ try {
93
+ runConfigData = JSON.parse(process.env.OW_RUN_CONFIG);
94
+ } catch (error) {
95
+ throw new Error(`Failed to parse OW_RUN_CONFIG: ${error.message}`);
96
+ }
91
97
  }
92
- const storageClass = global[storageName];
93
- if (!storageClass) {
94
- throw new Error(`Storage class ${storageName} not found`);
98
+ const context = runContext = new Core.AbstractContext({
99
+ source: { name: envConfig.source.name, config: envConfig.source.config || {} },
100
+ storage: { name: envConfig.storage.name, config: envConfig.storage.config || {} },
101
+ runConfig: runConfigData,
102
+ env: {
103
+ datamartId: process.env.OW_DATAMART_ID,
104
+ runId: process.env.OW_RUN_ID
105
+ }
106
+ });
107
+ const sourceName = context.sourceName;
108
+ const connectorBundle = global[sourceName];
109
+ const SourceClass = connectorBundle && connectorBundle[sourceName + "Source"];
110
+ let StorageClass;
111
+ if (isTestMode) {
112
+ StorageClass = Core.TestStorage;
113
+ } else {
114
+ const storageClassName = context.storageName + "Storage";
115
+ StorageClass = global[storageClassName];
116
+ if (!StorageClass) {
117
+ throw new Error(`Storage class "${storageClassName}" not found in global namespace`);
118
+ }
119
+ }
120
+ let customManifest = null;
121
+ if (process.env.OW_MANIFEST) {
122
+ try {
123
+ customManifest = JSON.parse(process.env.OW_MANIFEST);
124
+ } catch (error) {
125
+ throw new Error(`Failed to parse OW_MANIFEST: ${error.message}`);
126
+ }
127
+ }
128
+ const sourceOptions = isTestMode ? { ...testLimits, emitSample: true } : testLimits;
129
+ let source;
130
+ if (customManifest) {
131
+ const model = new Core.ManifestParser().parse(JSON.stringify(customManifest));
132
+ source = new Core.DeclarativeSource(context, model, sourceOptions);
133
+ } else if (SourceClass) {
134
+ source = new SourceClass(context);
135
+ } else if (connectorBundle && connectorBundle.manifest && connectorBundle.manifest.nodes) {
136
+ const model = new Core.ManifestParser().parse(JSON.stringify(connectorBundle.manifest));
137
+ source = new Core.DeclarativeSource(context, model, sourceOptions);
138
+ } else {
139
+ throw new Error(
140
+ `Source class "${sourceName}Source" not found and no declarative manifest for "${sourceName}"`
141
+ );
95
142
  }
96
- const source = new sourceClass[sourceName + "Source"](config);
97
- const connector = new sourceClass[sourceName + "Connector"](
98
- config,
99
- source,
100
- storageName + "Storage",
101
- runConfig
102
- );
143
+ const connector = new Core.AbstractConnector(context, source, StorageClass);
103
144
  await connector.run();
104
145
  }
105
146
  main().catch((error) => {
106
- const isWarning = (error == null ? void 0 : error.isWarning) === true;
107
- const detail = isWarning ? (error == null ? void 0 : error.message) ?? String(error) : (error == null ? void 0 : error.stack) ?? String(error);
108
- console.error(JSON.stringify({
109
- type: isWarning ? "addWarningToCurrentStatus" : "error",
110
- at: (/* @__PURE__ */ new Date()).toISOString(),
111
- [isWarning ? "warning" : "error"]: detail
112
- }));
147
+ const envelope = JSON.stringify(Core.RunFailureReport.toEnvelope(error));
148
+ console.error(runContext ? runContext.redactSecrets(envelope) : envelope);
113
149
  });
package/dist/connector-runner.js CHANGED
@@ -63,6 +63,7 @@ Object.keys(Connectors).forEach((key) => {
63
63
  global[key2] = connector[key2];
64
64
  });
65
65
  });
66
+ let runContext = null;
66
67
  async function main() {
67
68
  if (!process.env.OW_CONFIG) {
68
69
  throw new Error("OW_CONFIG environment variable is required");
@@ -73,40 +74,75 @@ async function main() {
73
74
  if (!process.env.OW_RUN_ID) {
74
75
  throw new Error("OW_RUN_ID environment variable is required");
75
76
  }
77
+ const isTestMode = Boolean(process.env.OW_TEST);
78
+ const testLimits = isTestMode ? {
79
+ maxRows: process.env.OW_TEST_MAX_ROWS ? parseInt(process.env.OW_TEST_MAX_ROWS, 10) : void 0,
80
+ maxPages: process.env.OW_TEST_MAX_PAGES ? parseInt(process.env.OW_TEST_MAX_PAGES, 10) : void 0,
81
+ maxSlices: process.env.OW_TEST_MAX_SLICES ? parseInt(process.env.OW_TEST_MAX_SLICES, 10) : 10
82
+ } : {};
76
83
  let envConfig;
77
84
  try {
78
85
  envConfig = JSON.parse(process.env.OW_CONFIG);
79
86
  } catch (error) {
80
87
  throw new Error(`Failed to parse OW_CONFIG: ${error.message}`);
81
88
  }
82
- const config = new Core.NodeJsConfig(envConfig);
83
- const runConfigJson = process.env.OW_RUN_CONFIG;
84
- const runConfig = runConfigJson ? new Core.AbstractRunConfig(JSON.parse(runConfigJson)) : new Core.AbstractRunConfig();
85
- const sourceName = config.getSourceName();
86
- const storageName = config.getStorageName();
87
- const sourceClass = global[sourceName];
88
- if (!sourceClass) {
89
- throw new Error(`Source class ${sourceName} not found`);
89
+ let runConfigData = null;
90
+ if (process.env.OW_RUN_CONFIG) {
91
+ try {
92
+ runConfigData = JSON.parse(process.env.OW_RUN_CONFIG);
93
+ } catch (error) {
94
+ throw new Error(`Failed to parse OW_RUN_CONFIG: ${error.message}`);
95
+ }
90
96
  }
91
- const storageClass = global[storageName];
92
- if (!storageClass) {
93
- throw new Error(`Storage class ${storageName} not found`);
97
+ const context = runContext = new Core.AbstractContext({
98
+ source: { name: envConfig.source.name, config: envConfig.source.config || {} },
99
+ storage: { name: envConfig.storage.name, config: envConfig.storage.config || {} },
100
+ runConfig: runConfigData,
101
+ env: {
102
+ datamartId: process.env.OW_DATAMART_ID,
103
+ runId: process.env.OW_RUN_ID
104
+ }
105
+ });
106
+ const sourceName = context.sourceName;
107
+ const connectorBundle = global[sourceName];
108
+ const SourceClass = connectorBundle && connectorBundle[sourceName + "Source"];
109
+ let StorageClass;
110
+ if (isTestMode) {
111
+ StorageClass = Core.TestStorage;
112
+ } else {
113
+ const storageClassName = context.storageName + "Storage";
114
+ StorageClass = global[storageClassName];
115
+ if (!StorageClass) {
116
+ throw new Error(`Storage class "${storageClassName}" not found in global namespace`);
117
+ }
118
+ }
119
+ let customManifest = null;
120
+ if (process.env.OW_MANIFEST) {
121
+ try {
122
+ customManifest = JSON.parse(process.env.OW_MANIFEST);
123
+ } catch (error) {
124
+ throw new Error(`Failed to parse OW_MANIFEST: ${error.message}`);
125
+ }
126
+ }
127
+ const sourceOptions = isTestMode ? { ...testLimits, emitSample: true } : testLimits;
128
+ let source;
129
+ if (customManifest) {
130
+ const model = new Core.ManifestParser().parse(JSON.stringify(customManifest));
131
+ source = new Core.DeclarativeSource(context, model, sourceOptions);
132
+ } else if (SourceClass) {
133
+ source = new SourceClass(context);
134
+ } else if (connectorBundle && connectorBundle.manifest && connectorBundle.manifest.nodes) {
135
+ const model = new Core.ManifestParser().parse(JSON.stringify(connectorBundle.manifest));
136
+ source = new Core.DeclarativeSource(context, model, sourceOptions);
137
+ } else {
138
+ throw new Error(
139
+ `Source class "${sourceName}Source" not found and no declarative manifest for "${sourceName}"`
140
+ );
94
141
  }
95
- const source = new sourceClass[sourceName + "Source"](config);
96
- const connector = new sourceClass[sourceName + "Connector"](
97
- config,
98
- source,
99
- storageName + "Storage",
100
- runConfig
101
- );
142
+ const connector = new Core.AbstractConnector(context, source, StorageClass);
102
143
  await connector.run();
103
144
  }
104
145
  main().catch((error) => {
105
- const isWarning = (error == null ? void 0 : error.isWarning) === true;
106
- const detail = isWarning ? (error == null ? void 0 : error.message) ?? String(error) : (error == null ? void 0 : error.stack) ?? String(error);
107
- console.error(JSON.stringify({
108
- type: isWarning ? "addWarningToCurrentStatus" : "error",
109
- at: (/* @__PURE__ */ new Date()).toISOString(),
110
- [isWarning ? "warning" : "error"]: detail
111
- }));
146
+ const envelope = JSON.stringify(Core.RunFailureReport.toEnvelope(error));
147
+ console.error(runContext ? runContext.redactSecrets(envelope) : envelope);
112
148
  });