@ingestron/pack-databricks 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/sql-data.js +22 -4
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -60,6 +60,9 @@ Used with the [ADF pack](https://github.com/ingestron/pack-adf)
|
|
|
60
60
|
(`packs: [databricks@1, adf@1]`), streaming tables read the files ADF lands,
|
|
61
61
|
from its `landing` folder.
|
|
62
62
|
|
|
63
|
+
Columns ADF adds to the landed rows (`audit_columns` in ADF 1.4) are read
|
|
64
|
+
with the files; timestamps, which ADF writes as text, are cast to `TIMESTAMP`.
|
|
65
|
+
|
|
63
66
|
## Dependencies
|
|
64
67
|
|
|
65
68
|
- [`@ingestron/definitions`](https://www.npmjs.com/package/@ingestron/definitions), a peer dependency: the resolved model.
|
package/dist/sql-data.js
CHANGED
|
@@ -46,13 +46,31 @@ function renamedColumns(table) {
|
|
|
46
46
|
return [];
|
|
47
47
|
return table.contract.columns.filter((column) => column.physicalName && column.physicalName !== column.name);
|
|
48
48
|
}
|
|
49
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* Timestamps the landed files carry beyond the contract (such as ADF's audit
|
|
51
|
+
* columns), which the step that adds them writes as ISO 8601 text.
|
|
52
|
+
*/
|
|
53
|
+
function textTimestamps(table) {
|
|
54
|
+
const extra = table.ingest?.output?.columns ?? [];
|
|
55
|
+
return extra.filter((column) => column.logicalType === "timestamp").map((column) => column.name);
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* `*`, or `* EXCEPT (<source names>), <source> AS <name>, …` for renamed
|
|
59
|
+
* columns, and `CAST(<name> AS TIMESTAMP) AS <name>` for timestamps written as text.
|
|
60
|
+
*/
|
|
50
61
|
function selectOf(table) {
|
|
51
62
|
const renamed = renamedColumns(table);
|
|
52
|
-
|
|
63
|
+
const timestamps = textTimestamps(table);
|
|
64
|
+
if (renamed.length === 0 && timestamps.length === 0)
|
|
53
65
|
return "*";
|
|
54
|
-
const sources =
|
|
55
|
-
|
|
66
|
+
const sources = [
|
|
67
|
+
...renamed.map((column) => quoteName(column.physicalName)),
|
|
68
|
+
...timestamps.map(quoteName),
|
|
69
|
+
];
|
|
70
|
+
const aliases = [
|
|
71
|
+
...renamed.map((column) => `${quoteName(column.physicalName)} AS ${quoteName(column.name)}`),
|
|
72
|
+
...timestamps.map((name) => `CAST(${quoteName(name)} AS TIMESTAMP) AS ${quoteName(name)}`),
|
|
73
|
+
];
|
|
56
74
|
return `* EXCEPT (${sources.join(", ")}), ${aliases.join(", ")}`;
|
|
57
75
|
}
|
|
58
76
|
/** The connection's `path` joined with the table's object. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ingestron/pack-databricks",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Databricks methods for Ingestron: generates a bundle with a Lakeflow Spark Declarative Pipeline.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "IntentLabs <hello@ingestron.io>",
|
|
@@ -34,11 +34,11 @@
|
|
|
34
34
|
"yaml": "2.9.1"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
|
-
"@ingestron/definitions": "^1.
|
|
37
|
+
"@ingestron/definitions": "^1.2.0"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"@ingestron/config": "^1.0.0",
|
|
41
|
-
"@ingestron/definitions": "^1.
|
|
41
|
+
"@ingestron/definitions": "^1.5.0",
|
|
42
42
|
"@microsoft/api-extractor": "7.59.4",
|
|
43
43
|
"@types/node": "22.20.5",
|
|
44
44
|
"ajv": "8.20.0",
|