@meshxdata/fops 0.1.36 → 0.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +207 -0
- package/fops.mjs +37 -14
- package/package.json +1 -1
- package/src/agent/llm.js +2 -0
- package/src/auth/azure.js +92 -0
- package/src/auth/cloudflare.js +125 -0
- package/src/auth/index.js +2 -0
- package/src/commands/index.js +8 -4
- package/src/commands/lifecycle.js +31 -10
- package/src/plugins/bundled/fops-plugin-azure/index.js +44 -2896
- package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks.js +130 -2
- package/src/plugins/bundled/fops-plugin-azure/lib/azure-auth.js +497 -0
- package/src/plugins/bundled/fops-plugin-azure/lib/azure-helpers.js +51 -13
- package/src/plugins/bundled/fops-plugin-azure/lib/azure-ops.js +206 -52
- package/src/plugins/bundled/fops-plugin-azure/lib/azure-provision.js +128 -34
- package/src/plugins/bundled/fops-plugin-azure/lib/azure-shared-cache.js +1 -1
- package/src/plugins/bundled/fops-plugin-azure/lib/azure-sync.js +4 -4
- package/src/plugins/bundled/fops-plugin-azure/lib/azure.js +2 -2
- package/src/plugins/bundled/fops-plugin-azure/lib/commands/fleet-cmds.js +254 -0
- package/src/plugins/bundled/fops-plugin-azure/lib/commands/infra-cmds.js +894 -0
- package/src/plugins/bundled/fops-plugin-azure/lib/commands/test-cmds.js +314 -0
- package/src/plugins/bundled/fops-plugin-azure/lib/commands/vm-cmds.js +893 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/dai-backend.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/dai-frontend.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-backend.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-frontend.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-hive.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-kafka.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-meltano.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-mlflow.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-opa.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-processor.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-scheduler.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-storage-engine.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-trino.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/apps/foundation-watcher.yaml +13 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/config/repository.yaml +66 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/kustomization.yaml +30 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/acr-webhook-controller.yaml +63 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/externalsecrets.yaml +15 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/istio.yaml +42 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/kafka.yaml +15 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/kube-reflector.yaml +33 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/kubecost.yaml +12 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/nats-server.yaml +15 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/prometheus-agent.yaml +34 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/reloader.yaml +12 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/spark.yaml +112 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/tailscale.yaml +67 -0
- package/src/plugins/bundled/fops-plugin-azure/templates/cluster/operator/vertical-pod-autoscaler.yaml +15 -0
- package/src/plugins/bundled/fops-plugin-file/demo/orders_renamed.aligned.csv +1 -1
- package/src/plugins/bundled/fops-plugin-file/index.js +81 -12
- package/src/plugins/bundled/fops-plugin-file/lib/match.js +133 -15
- package/src/plugins/bundled/fops-plugin-file/lib/report.js +3 -0
- package/src/plugins/bundled/fops-plugin-foundation/index.js +26 -6
- package/src/plugins/bundled/fops-plugin-foundation/lib/client.js +9 -5
- package/src/plugins/bundled/fops-plugin-foundation-graphql/lib/graphql/resolvers/data-product.js +32 -0
- package/src/plugins/bundled/fops-plugin-foundation-graphql/lib/graphql/schema.js +20 -1
- package/src/plugins/loader.js +23 -6
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
|
2
|
+
kind: Kustomization
|
|
3
|
+
metadata:
|
|
4
|
+
name: tailscale-operator
|
|
5
|
+
namespace: flux-system
|
|
6
|
+
spec:
|
|
7
|
+
interval: 1m0s
|
|
8
|
+
sourceRef:
|
|
9
|
+
kind: GitRepository
|
|
10
|
+
name: flux-system
|
|
11
|
+
path: ./infrastructure/tailscale/operator
|
|
12
|
+
prune: false
|
|
13
|
+
dependsOn:
|
|
14
|
+
- name: acr-webhook-controller
|
|
15
|
+
namespace: flux-system
|
|
16
|
+
patches:
|
|
17
|
+
- target:
|
|
18
|
+
kind: HelmRelease
|
|
19
|
+
name: tailscale-operator
|
|
20
|
+
namespace: flux-system
|
|
21
|
+
patch: |-
|
|
22
|
+
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
23
|
+
kind: HelmRelease
|
|
24
|
+
metadata:
|
|
25
|
+
name: tailscale-operator
|
|
26
|
+
namespace: flux-system
|
|
27
|
+
spec:
|
|
28
|
+
values:
|
|
29
|
+
oauth:
|
|
30
|
+
clientId: "{{TAILSCALE_CLIENT_ID}}"
|
|
31
|
+
clientSecret: "{{TAILSCALE_CLIENT_SECRET}}"
|
|
32
|
+
operatorConfig:
|
|
33
|
+
hostname: "{{CLUSTER_NAME}}"
|
|
34
|
+
---
|
|
35
|
+
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
|
36
|
+
kind: Kustomization
|
|
37
|
+
metadata:
|
|
38
|
+
name: tailscale-connector
|
|
39
|
+
namespace: flux-system
|
|
40
|
+
spec:
|
|
41
|
+
interval: 1m0s
|
|
42
|
+
sourceRef:
|
|
43
|
+
kind: GitRepository
|
|
44
|
+
name: flux-system
|
|
45
|
+
path: ./infrastructure/tailscale/connector
|
|
46
|
+
prune: false
|
|
47
|
+
dependsOn:
|
|
48
|
+
- name: tailscale-operator
|
|
49
|
+
patches:
|
|
50
|
+
- target:
|
|
51
|
+
kind: Connector
|
|
52
|
+
name: default-connector
|
|
53
|
+
namespace: tailscale
|
|
54
|
+
patch: |-
|
|
55
|
+
- op: replace
|
|
56
|
+
path: /metadata/name
|
|
57
|
+
value: "{{CLUSTER_NAME}}"
|
|
58
|
+
- op: replace
|
|
59
|
+
path: /spec/hostname
|
|
60
|
+
value: "{{CLUSTER_NAME}}"
|
|
61
|
+
- op: replace
|
|
62
|
+
path: /spec/exitNode
|
|
63
|
+
value: true
|
|
64
|
+
- op: replace
|
|
65
|
+
path: /spec/subnetRouter/advertiseRoutes
|
|
66
|
+
value:
|
|
67
|
+
- 10.50.0.0/16
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
|
2
|
+
kind: Kustomization
|
|
3
|
+
metadata:
|
|
4
|
+
name: vertical-pod-autoscaler
|
|
5
|
+
namespace: flux-system
|
|
6
|
+
spec:
|
|
7
|
+
interval: 1m0s
|
|
8
|
+
sourceRef:
|
|
9
|
+
kind: GitRepository
|
|
10
|
+
name: flux-system
|
|
11
|
+
path: ./infrastructure/vertical-pod-autoscaler/overlays/{{OVERLAY}}
|
|
12
|
+
prune: false
|
|
13
|
+
dependsOn:
|
|
14
|
+
- name: acr-webhook-controller
|
|
15
|
+
namespace: flux-system
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
order_id,customer_id,price,ccy,order_date,status,region
|
|
2
2
|
4001,8001,99.00,USD,2024-03-01,completed,north_america
|
|
3
3
|
4002,8002,145.75,EUR,2024-03-02,pending,europe
|
|
4
4
|
4003,8003,280.00,USD,2024-03-03,completed,asia_pacific
|
|
@@ -447,20 +447,89 @@ export function register(api) {
|
|
|
447
447
|
}
|
|
448
448
|
|
|
449
449
|
// ── 9. Show column mapping (if available) ──────────────────────
|
|
450
|
-
|
|
450
|
+
let currentMappings = [...(selectedRef.columnMappings || [])];
|
|
451
|
+
if (currentMappings.length > 0) {
|
|
451
452
|
const refData2 = references.find((r) => r.id === selectedRef.id);
|
|
452
453
|
if (refData2) {
|
|
453
454
|
const refTag = `${selectedRef.name} (${selectedRef.id.slice(0, 8)})`;
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
455
|
+
|
|
456
|
+
const showMappingTable = () => {
|
|
457
|
+
console.log("");
|
|
458
|
+
console.log(formatColumnMapping(
|
|
459
|
+
currentMappings,
|
|
460
|
+
candResult.headers,
|
|
461
|
+
refData2.headers,
|
|
462
|
+
pathMod.basename(candidate),
|
|
463
|
+
refTag,
|
|
464
|
+
chalk,
|
|
465
|
+
));
|
|
466
|
+
};
|
|
467
|
+
showMappingTable();
|
|
468
|
+
|
|
469
|
+
// ── 9b. Interactive mapping adjustment (TTY only) ──────────
|
|
470
|
+
if (isTTY && !opts.json) {
|
|
471
|
+
let _selectOpt;
|
|
472
|
+
const _cpPath = api.getCliPath?.("src", "ui", "confirm.js");
|
|
473
|
+
if (_cpPath) {
|
|
474
|
+
const { pathToFileURL: _pu } = await import("node:url");
|
|
475
|
+
({ selectOption: _selectOpt } = await import(_pu(_cpPath).href));
|
|
476
|
+
} else {
|
|
477
|
+
({ selectOption: _selectOpt } = await import("../../../ui/confirm.js"));
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
let adjusting = true;
|
|
481
|
+
while (adjusting) {
|
|
482
|
+
const choice = await _selectOpt("Column mappings:", [
|
|
483
|
+
{ label: "Continue to validation", value: "continue" },
|
|
484
|
+
{ label: "Adjust a mapping", value: "adjust" },
|
|
485
|
+
]);
|
|
486
|
+
if (choice !== "adjust") { adjusting = false; break; }
|
|
487
|
+
|
|
488
|
+
// Step 1: pick candidate column to remap
|
|
489
|
+
const mappedCandSet = new Set(currentMappings.map((m) => m.candidateCol));
|
|
490
|
+
const candOptions = [
|
|
491
|
+
...currentMappings.map((m) => {
|
|
492
|
+
const sim = m.similarity == null ? "manual"
|
|
493
|
+
: m.candidateCol.toLowerCase() === m.referenceCol.toLowerCase() ? "exact"
|
|
494
|
+
: m.similarity.toFixed(2);
|
|
495
|
+
return { label: `${m.candidateCol.padEnd(22)} → ${m.referenceCol} (${sim})`, value: m.candidateCol };
|
|
496
|
+
}),
|
|
497
|
+
...candResult.headers
|
|
498
|
+
.filter((h) => !mappedCandSet.has(h))
|
|
499
|
+
.map((h) => ({ label: `${h.padEnd(22)} → (unmapped)`, value: h })),
|
|
500
|
+
];
|
|
501
|
+
|
|
502
|
+
const chosenCand = await _selectOpt("Which column to remap:", candOptions);
|
|
503
|
+
if (chosenCand == null) continue;
|
|
504
|
+
|
|
505
|
+
// Step 2: pick reference field — show all, marking already-mapped ones
|
|
506
|
+
const refFieldOwner = new Map(
|
|
507
|
+
currentMappings.filter((m) => m.candidateCol !== chosenCand).map((m) => [m.referenceCol, m.candidateCol]),
|
|
508
|
+
);
|
|
509
|
+
const refFieldOptions = [
|
|
510
|
+
...refData2.headers.map((h) => {
|
|
511
|
+
const owner = refFieldOwner.get(h);
|
|
512
|
+
return owner
|
|
513
|
+
? { label: `${h} (currently: ${owner})`, value: h }
|
|
514
|
+
: { label: h, value: h };
|
|
515
|
+
}),
|
|
516
|
+
{ label: "— leave unmapped —", value: "__none__" },
|
|
517
|
+
];
|
|
518
|
+
|
|
519
|
+
const chosenRef = await _selectOpt(`Map "${chosenCand}" to:`, refFieldOptions);
|
|
520
|
+
if (chosenRef == null) continue;
|
|
521
|
+
|
|
522
|
+
// Apply — remove chosen candidate's old mapping and displace any
|
|
523
|
+
// existing owner of the target reference field
|
|
524
|
+
currentMappings = currentMappings.filter(
|
|
525
|
+
(m) => m.candidateCol !== chosenCand && m.referenceCol !== chosenRef,
|
|
526
|
+
);
|
|
527
|
+
if (chosenRef !== "__none__") {
|
|
528
|
+
currentMappings.push({ candidateCol: chosenCand, referenceCol: chosenRef, similarity: null });
|
|
529
|
+
}
|
|
530
|
+
showMappingTable();
|
|
531
|
+
}
|
|
532
|
+
}
|
|
464
533
|
}
|
|
465
534
|
}
|
|
466
535
|
|
|
@@ -481,7 +550,7 @@ export function register(api) {
|
|
|
481
550
|
// names to their matched reference names so schema/type checks are meaningful.
|
|
482
551
|
let candHeadersForValidation = candResult.headers;
|
|
483
552
|
let candTypesForValidation = candResult.types;
|
|
484
|
-
const colMap =
|
|
553
|
+
const colMap = currentMappings;
|
|
485
554
|
if (colMap?.length > 0) {
|
|
486
555
|
const translatedHeaders = [];
|
|
487
556
|
const translatedTypes = {};
|
|
@@ -44,6 +44,69 @@ export function buildFingerprint({ headers, types, name }) {
|
|
|
44
44
|
return { columnText, typeText, nameText };
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
// ── Levenshtein ─────────────────────────────────────────────────────────────
|
|
48
|
+
|
|
49
|
+
function _levenshteinDist(a, b) {
|
|
50
|
+
const m = a.length, n = b.length;
|
|
51
|
+
const dp = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
|
|
52
|
+
for (let i = 0; i <= m; i++) dp[i][0] = i;
|
|
53
|
+
for (let j = 0; j <= n; j++) dp[0][j] = j;
|
|
54
|
+
for (let i = 1; i <= m; i++) {
|
|
55
|
+
for (let j = 1; j <= n; j++) {
|
|
56
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
57
|
+
dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + cost);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return dp[m][n];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function _levenshteinRatio(a, b) {
|
|
64
|
+
if (a === b) return 1;
|
|
65
|
+
if (!a || !b) return 0;
|
|
66
|
+
return 1 - _levenshteinDist(a, b) / Math.max(a.length, b.length);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Greedy bipartite column matching using Levenshtein ratio + containment boost.
|
|
71
|
+
* Same output shape as greedyColumnMatch so callers are interchangeable.
|
|
72
|
+
*
|
|
73
|
+
* Containment boost: single-token candidate = last segment of a compound reference
|
|
74
|
+
* (e.g. "id" → "order_id", "date" → "order_date") gets +0.30 so it beats
|
|
75
|
+
* weaker multi-token candidates claiming the same field.
|
|
76
|
+
*
|
|
77
|
+
* @param {string[]} candHeaders
|
|
78
|
+
* @param {string[]} refHeaders
|
|
79
|
+
* @param {object} [opts]
|
|
80
|
+
* @param {number} [opts.minSim=0.45]
|
|
81
|
+
* @returns {{score: number, mappings: Array<{candIdx: number, refIdx: number, sim: number}>}}
|
|
82
|
+
*/
|
|
83
|
+
export function levenshteinColumnMatch(candHeaders, refHeaders, opts = {}) {
|
|
84
|
+
const minSim = opts.minSim ?? 0.45;
|
|
85
|
+
const M = candHeaders.length, N = refHeaders.length;
|
|
86
|
+
if (M === 0 || N === 0) return { score: 0, mappings: [] };
|
|
87
|
+
|
|
88
|
+
const norm = (s) => s.toLowerCase().replace(/[_\-\s]+/g, "_");
|
|
89
|
+
|
|
90
|
+
const simMatrix = [];
|
|
91
|
+
for (let i = 0; i < M; i++) {
|
|
92
|
+
simMatrix[i] = [];
|
|
93
|
+
const cn = norm(candHeaders[i]);
|
|
94
|
+
const cParts = cn.split("_").filter(Boolean);
|
|
95
|
+
for (let j = 0; j < N; j++) {
|
|
96
|
+
const rn = norm(refHeaders[j]);
|
|
97
|
+
const rParts = rn.split("_").filter(Boolean);
|
|
98
|
+
let sim = _levenshteinRatio(cn, rn);
|
|
99
|
+
// Containment: single-token candidate IS the last segment of the reference
|
|
100
|
+
if (cParts.length === 1 && rParts.length > 1 && rParts[rParts.length - 1] === cParts[0]) {
|
|
101
|
+
sim = Math.min(1, sim + 0.30);
|
|
102
|
+
}
|
|
103
|
+
simMatrix[i][j] = sim;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return greedyColumnMatch(simMatrix, { minSim });
|
|
108
|
+
}
|
|
109
|
+
|
|
47
110
|
// ── Jaccard Overlap ─────────────────────────────────────────────────────────
|
|
48
111
|
|
|
49
112
|
/**
|
|
@@ -167,8 +230,11 @@ function normalizeColumnName(name) {
|
|
|
167
230
|
.map((p) => p.toLowerCase());
|
|
168
231
|
if (parts.length === 0) return name || "";
|
|
169
232
|
if (parts.length === 1) return parts[0];
|
|
170
|
-
// Repeat the last token
|
|
171
|
-
|
|
233
|
+
// Repeat the last token 3 extra times so the semantic role (suffix) dominates
|
|
234
|
+
// over the entity prefix. e.g. "order_status" → "order status status status status"
|
|
235
|
+
// vs "order_id" → "order id id id id" — "status" and "id" are now discriminating.
|
|
236
|
+
const last = parts[parts.length - 1];
|
|
237
|
+
return parts.join(" ") + " " + last + " " + last + " " + last;
|
|
172
238
|
}
|
|
173
239
|
|
|
174
240
|
/**
|
|
@@ -194,21 +260,27 @@ export function buildSimilarityMatrix(candVecs, refVecs) {
|
|
|
194
260
|
/**
|
|
195
261
|
* Greedy bipartite matching on a similarity matrix.
|
|
196
262
|
* Sort all (i,j,sim) triples descending, greedily assign unmatched pairs.
|
|
263
|
+
* Pairs below minSim are left unmatched (shown as unmapped in the display).
|
|
197
264
|
*
|
|
198
265
|
* @param {number[][]} simMatrix - M×N cosine similarity matrix
|
|
266
|
+
* @param {object} [opts]
|
|
267
|
+
* @param {number} [opts.minSim=0.45] - Minimum similarity to accept a pairing
|
|
199
268
|
* @returns {{score: number, mappings: Array<{candIdx: number, refIdx: number, sim: number}>}}
|
|
200
269
|
*/
|
|
201
|
-
export function greedyColumnMatch(simMatrix) {
|
|
270
|
+
export function greedyColumnMatch(simMatrix, opts = {}) {
|
|
271
|
+
const minSim = opts.minSim ?? 0.45;
|
|
202
272
|
const M = simMatrix.length;
|
|
203
273
|
if (M === 0) return { score: 0, mappings: [] };
|
|
204
274
|
const N = simMatrix[0].length;
|
|
205
275
|
if (N === 0) return { score: 0, mappings: [] };
|
|
206
276
|
|
|
207
|
-
// Collect all pairs
|
|
277
|
+
// Collect all pairs above the minimum threshold
|
|
208
278
|
const pairs = [];
|
|
209
279
|
for (let i = 0; i < M; i++) {
|
|
210
280
|
for (let j = 0; j < N; j++) {
|
|
211
|
-
|
|
281
|
+
if (simMatrix[i][j] >= minSim) {
|
|
282
|
+
pairs.push({ candIdx: i, refIdx: j, sim: simMatrix[i][j] });
|
|
283
|
+
}
|
|
212
284
|
}
|
|
213
285
|
}
|
|
214
286
|
pairs.sort((a, b) => b.sim - a.sim);
|
|
@@ -337,16 +409,46 @@ export function rankMatches(candidate, references, opts = {}) {
|
|
|
337
409
|
.sort((a, b) => b.jaccardScore - a.jaccardScore);
|
|
338
410
|
|
|
339
411
|
if (!hasEmb && !hasColEmb) {
|
|
340
|
-
// Jaccard
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
412
|
+
// Jaccard + Levenshtein column match — no ONNX needed.
|
|
413
|
+
// Levenshtein catches abbreviations (cust_id ↔ customer_id) and containment
|
|
414
|
+
// (id ↔ order_id) that pure Jaccard misses.
|
|
415
|
+
const levList = references.map((ref) => {
|
|
416
|
+
const { score, mappings } = levenshteinColumnMatch(candidate.headers, ref.headers);
|
|
417
|
+
const columnMappings = mappings.map((m) => ({
|
|
418
|
+
candidateCol: candidate.headers[m.candIdx],
|
|
419
|
+
referenceCol: ref.headers[m.refIdx],
|
|
420
|
+
similarity: m.sim,
|
|
421
|
+
}));
|
|
422
|
+
return { id: ref.id, levScore: score, columnMappings };
|
|
423
|
+
}).sort((a, b) => b.levScore - a.levScore);
|
|
424
|
+
|
|
425
|
+
const fused = mergeRRF([
|
|
426
|
+
jaccardList.map((r) => ({ id: r.id })),
|
|
427
|
+
levList.map((r) => ({ id: r.id })),
|
|
428
|
+
], topK);
|
|
429
|
+
|
|
430
|
+
const jaccardMap = new Map(jaccardList.map((r) => [r.id, r]));
|
|
431
|
+
const levMap = new Map(levList.map((r) => [r.id, r]));
|
|
432
|
+
const refById = new Map(references.map((r) => [r.id, r]));
|
|
433
|
+
|
|
434
|
+
const W_JAC = 0.40, W_LEV = 0.60;
|
|
435
|
+
|
|
436
|
+
return fused.map((r) => {
|
|
437
|
+
const j = jaccardMap.get(r.id) || { overlap: { count: 0, total: 0 }, jaccardScore: 0 };
|
|
438
|
+
const l = levMap.get(r.id) || { levScore: 0, columnMappings: [] };
|
|
439
|
+
const ref = refById.get(r.id);
|
|
440
|
+
const quality = W_JAC * j.jaccardScore + W_LEV * l.levScore;
|
|
441
|
+
return {
|
|
442
|
+
id: r.id,
|
|
443
|
+
name: ref?.name || r.id,
|
|
444
|
+
entityType: ref?.entityType,
|
|
445
|
+
score: Math.round(quality * 100),
|
|
446
|
+
overlapCount: j.overlap.count,
|
|
447
|
+
overlapTotal: j.overlap.total,
|
|
448
|
+
columnMappings: l.columnMappings,
|
|
449
|
+
signals: { jaccard: j.jaccardScore, levenshtein: l.levScore },
|
|
450
|
+
};
|
|
451
|
+
});
|
|
350
452
|
}
|
|
351
453
|
|
|
352
454
|
// ── Column-level pipeline (v3) ──────────────────────────────────────────
|
|
@@ -373,6 +475,22 @@ function _rankWithColumnEmbeddings(candidate, references, jaccardList, topK) {
|
|
|
373
475
|
|
|
374
476
|
if (candColVecs.length > 0 && refColVecs.length > 0) {
|
|
375
477
|
const simMatrix = buildSimilarityMatrix(candColVecs, refColVecs);
|
|
478
|
+
|
|
479
|
+
// Containment boost: a single-token candidate that equals the last segment of a
|
|
480
|
+
// compound reference name is almost certainly the right match (e.g. "id" → "order_id",
|
|
481
|
+
// "date" → "order_date"). Boost its score so the greedy assigns it before a weaker
|
|
482
|
+
// multi-token candidate (e.g. "order_status") can claim the reference field.
|
|
483
|
+
for (let i = 0; i < candidate.headers.length; i++) {
|
|
484
|
+
const cParts = candidate.headers[i].toLowerCase().split(/[_\-\s]+/).filter(Boolean);
|
|
485
|
+
if (cParts.length !== 1) continue; // only single-token names
|
|
486
|
+
for (let j = 0; j < ref.headers.length; j++) {
|
|
487
|
+
const rParts = ref.headers[j].toLowerCase().split(/[_\-\s]+/).filter(Boolean);
|
|
488
|
+
if (rParts.length > 1 && rParts[rParts.length - 1] === cParts[0]) {
|
|
489
|
+
simMatrix[i][j] = Math.min(1, simMatrix[i][j] + 0.15);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
376
494
|
colMatch = greedyColumnMatch(simMatrix);
|
|
377
495
|
typeCompat = typeCompatibilityScore(
|
|
378
496
|
candidate.types, ref.types,
|
|
@@ -136,6 +136,9 @@ export function formatColumnMapping(mappings, candHeaders, refHeaders, candLabel
|
|
|
136
136
|
if (m.candidateCol.toLowerCase() === m.referenceCol.toLowerCase()) {
|
|
137
137
|
matchLabel = "exact";
|
|
138
138
|
color = chalk.green;
|
|
139
|
+
} else if (m.similarity == null) {
|
|
140
|
+
matchLabel = "manual";
|
|
141
|
+
color = chalk.cyan;
|
|
139
142
|
} else if (m.similarity >= 0.85) {
|
|
140
143
|
matchLabel = m.similarity.toFixed(2);
|
|
141
144
|
color = chalk.green;
|
|
@@ -995,14 +995,22 @@ app.run()
|
|
|
995
995
|
|
|
996
996
|
let apiUrl = opts.url || process.env.FOPS_API_URL || "http://127.0.0.1:9001";
|
|
997
997
|
|
|
998
|
-
// Current fops version
|
|
998
|
+
// Current fops version — resolve from the running binary's location
|
|
999
999
|
let fopsVersion = "0.0.0";
|
|
1000
1000
|
try {
|
|
1001
|
-
|
|
1002
|
-
const fopsRoot2 = dirname(realpathSync(
|
|
1001
|
+
// process.argv[1] is the path to fops.mjs; package.json sits next to it
|
|
1002
|
+
const fopsRoot2 = dirname(realpathSync(process.argv[1]));
|
|
1003
1003
|
const pkgJson = JSON.parse(readFileSync(join(fopsRoot2, "package.json"), "utf8"));
|
|
1004
1004
|
fopsVersion = pkgJson.version || "0.0.0";
|
|
1005
|
-
} catch {
|
|
1005
|
+
} catch {
|
|
1006
|
+
// Fallback: try the plugins node_modules path
|
|
1007
|
+
try {
|
|
1008
|
+
const pluginsNodeModules2 = join(homedir(), ".fops", "plugins", "node_modules");
|
|
1009
|
+
const fopsRoot2 = dirname(realpathSync(pluginsNodeModules2));
|
|
1010
|
+
const pkgJson = JSON.parse(readFileSync(join(fopsRoot2, "package.json"), "utf8"));
|
|
1011
|
+
fopsVersion = pkgJson.version || "0.0.0";
|
|
1012
|
+
} catch { /* stay at 0.0.0 */ }
|
|
1013
|
+
}
|
|
1006
1014
|
|
|
1007
1015
|
// Resolve icon
|
|
1008
1016
|
let iconPath = "";
|
|
@@ -1449,6 +1457,8 @@ menu.addItem(NSMenuItem.separator())
|
|
|
1449
1457
|
let updateItem = NSMenuItem(title: "", action: #selector(AppDelegate.runUpdate), keyEquivalent: "")
|
|
1450
1458
|
updateItem.isHidden = true
|
|
1451
1459
|
menu.addItem(updateItem)
|
|
1460
|
+
let checkUpdateItem = NSMenuItem(title: "Check for updates", action: #selector(AppDelegate.checkForUpdateManual), keyEquivalent: "")
|
|
1461
|
+
menu.addItem(checkUpdateItem)
|
|
1452
1462
|
|
|
1453
1463
|
menu.addItem(NSMenuItem.separator())
|
|
1454
1464
|
menu.addItem(NSMenuItem(title: "Quit", action: #selector(NSApplication.terminate(_:)), keyEquivalent: "q"))
|
|
@@ -1467,9 +1477,9 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate {
|
|
|
1467
1477
|
refresh()
|
|
1468
1478
|
let pollTimer = Timer(timeInterval: 8, repeats: true) { _ in self.refresh() }
|
|
1469
1479
|
RunLoop.main.add(pollTimer, forMode: .common)
|
|
1470
|
-
// Check for updates on launch, then every
|
|
1480
|
+
// Check for updates on launch, then every 15 minutes
|
|
1471
1481
|
checkForUpdate()
|
|
1472
|
-
let updateTimer = Timer(timeInterval:
|
|
1482
|
+
let updateTimer = Timer(timeInterval: 900, repeats: true) { _ in self.checkForUpdate() }
|
|
1473
1483
|
RunLoop.main.add(updateTimer, forMode: .common)
|
|
1474
1484
|
}
|
|
1475
1485
|
|
|
@@ -1526,6 +1536,16 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate {
|
|
|
1526
1536
|
RunLoop.main.add(pulse, forMode: .common)
|
|
1527
1537
|
}
|
|
1528
1538
|
|
|
1539
|
+
@objc func checkForUpdateManual() {
|
|
1540
|
+
checkUpdateItem.title = "Checking…"
|
|
1541
|
+
checkUpdateItem.isEnabled = false
|
|
1542
|
+
checkForUpdate()
|
|
1543
|
+
DispatchQueue.main.asyncAfter(deadline: .now() + 3) {
|
|
1544
|
+
checkUpdateItem.title = "Check for updates"
|
|
1545
|
+
checkUpdateItem.isEnabled = true
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1529
1549
|
@objc func runUpdate() {
|
|
1530
1550
|
updateItem.title = "⬆ Updating…"
|
|
1531
1551
|
updateItem.isEnabled = false
|
|
@@ -90,17 +90,21 @@ function sleep(ms) {
|
|
|
90
90
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
/** Like request() but retries on network errors and 5xx responses.
|
|
93
|
+
/** Like request() but retries on network errors and 5xx responses.
|
|
94
|
+
* Non-idempotent methods (POST, PUT, PATCH, DELETE) are never retried on 5xx
|
|
95
|
+
* to avoid duplicate writes or zombie state (e.g. partial Iceberg table creation). */
|
|
94
96
|
async function requestWithRetries(method, url, headers, body, timeoutMs, maxRetries = RETRY_COUNT, delayMs = RETRY_DELAY_MS) {
|
|
95
|
-
|
|
97
|
+
const idempotent = method === "GET" || method === "HEAD" || method === "OPTIONS";
|
|
98
|
+
const effectiveMaxRetries = idempotent ? maxRetries : 1;
|
|
99
|
+
for (let attempt = 1; attempt <= effectiveMaxRetries; attempt++) {
|
|
96
100
|
let res;
|
|
97
101
|
try {
|
|
98
102
|
res = await request(method, url, headers, body, timeoutMs);
|
|
99
103
|
} catch (e) {
|
|
100
|
-
if (attempt <
|
|
101
|
-
throw new Error(`Request failed after ${
|
|
104
|
+
if (attempt < effectiveMaxRetries) { await sleep(delayMs); continue; }
|
|
105
|
+
throw new Error(`Request failed after ${attempt} attempts: ${e.message}`);
|
|
102
106
|
}
|
|
103
|
-
if (res.status >= 500 && attempt <
|
|
107
|
+
if (res.status >= 500 && attempt < effectiveMaxRetries) {
|
|
104
108
|
await sleep(delayMs);
|
|
105
109
|
continue;
|
|
106
110
|
}
|
package/src/plugins/bundled/fops-plugin-foundation-graphql/lib/graphql/resolvers/data-product.js
CHANGED
|
@@ -1,6 +1,38 @@
|
|
|
1
1
|
import { parseListResponse } from "../../../../fops-plugin-foundation/lib/api-spec.js";
|
|
2
2
|
|
|
3
3
|
export const dataProductResolvers = {
|
|
4
|
+
Mutation: {
|
|
5
|
+
async updateDataProductSchema(_root, { identifier, input }, { client }) {
|
|
6
|
+
const body = {
|
|
7
|
+
details: {
|
|
8
|
+
...(input.dataProductType != null && { data_product_type: input.dataProductType }),
|
|
9
|
+
fields: input.fields.map((f) => ({
|
|
10
|
+
name: f.name,
|
|
11
|
+
description: f.description ?? null,
|
|
12
|
+
primary: f.primary ?? false,
|
|
13
|
+
optional: f.optional ?? false,
|
|
14
|
+
data_type: {
|
|
15
|
+
meta: {},
|
|
16
|
+
column_type: f.dataType ?? "VARCHAR",
|
|
17
|
+
},
|
|
18
|
+
classification: f.classification ?? "internal",
|
|
19
|
+
sensitivity: f.sensitivity ?? null,
|
|
20
|
+
tags: f.tags ?? [],
|
|
21
|
+
})),
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
const res = await client.put(`/data/data_product/schema?identifier=${identifier}`, body);
|
|
25
|
+
const details = res?.details ?? res?.schema ?? res ?? null;
|
|
26
|
+
const columns = (res?.columns || res?.fields || input.fields).map((f) => ({
|
|
27
|
+
name: f.name || f.column_name,
|
|
28
|
+
dataType: f.data_type?.column_type ?? f.dataType ?? null,
|
|
29
|
+
primary: f.primary ?? false,
|
|
30
|
+
nullable: !(f.optional ?? false),
|
|
31
|
+
}));
|
|
32
|
+
return { details, columns };
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
|
|
4
36
|
Query: {
|
|
5
37
|
async dataProducts(_root, _args, { client }) {
|
|
6
38
|
const res = await client.get("/data/data_product/list?per_page=200");
|
|
@@ -1,11 +1,30 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* GraphQL SDL for the Foundation data mesh API.
|
|
3
|
-
* Read-only in v1 — no mutations.
|
|
4
3
|
*/
|
|
5
4
|
|
|
6
5
|
export const typeDefs = /* GraphQL */ `
|
|
7
6
|
scalar JSON
|
|
8
7
|
|
|
8
|
+
type Mutation {
|
|
9
|
+
updateDataProductSchema(identifier: ID!, input: DataProductSchemaInput!): DataProductSchema
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
input DataProductSchemaInput {
|
|
13
|
+
dataProductType: String
|
|
14
|
+
fields: [SchemaFieldInput!]!
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
input SchemaFieldInput {
|
|
18
|
+
name: String!
|
|
19
|
+
description: String
|
|
20
|
+
primary: Boolean
|
|
21
|
+
optional: Boolean
|
|
22
|
+
dataType: String
|
|
23
|
+
classification: String
|
|
24
|
+
sensitivity: String
|
|
25
|
+
tags: [String!]
|
|
26
|
+
}
|
|
27
|
+
|
|
9
28
|
type Query {
|
|
10
29
|
meshes: [Mesh!]!
|
|
11
30
|
mesh(identifier: ID!): Mesh
|
package/src/plugins/loader.js
CHANGED
|
@@ -156,21 +156,38 @@ function parseFrontmatter(content) {
|
|
|
156
156
|
// Alias for backward compatibility
|
|
157
157
|
const parseSkillFrontmatter = parseFrontmatter;
|
|
158
158
|
|
|
159
|
+
/**
|
|
160
|
+
* Plugins that are disabled by default and must be explicitly enabled via
|
|
161
|
+
* `fops plugin enable <id>` or by setting `enabled: true` in ~/.fops.json.
|
|
162
|
+
*/
|
|
163
|
+
const DEFAULT_DISABLED = new Set([
|
|
164
|
+
"fops-plugin-teleport",
|
|
165
|
+
"coda",
|
|
166
|
+
"fops-plugin-github-roles",
|
|
167
|
+
"fops-plugin-ecr",
|
|
168
|
+
"fops-plugin-vm-users",
|
|
169
|
+
"fops-plugin-aws",
|
|
170
|
+
"fops-plugin-dai-ttyd",
|
|
171
|
+
]);
|
|
172
|
+
|
|
159
173
|
/**
|
|
160
174
|
* Check if a plugin is enabled in ~/.fops.json.
|
|
161
|
-
*
|
|
175
|
+
* Plugins in DEFAULT_DISABLED are off unless explicitly enabled.
|
|
176
|
+
* All other plugins are on unless explicitly disabled.
|
|
162
177
|
*/
|
|
163
178
|
function isPluginEnabled(pluginId) {
|
|
164
179
|
try {
|
|
165
180
|
const fopsConfig = path.join(os.homedir(), ".fops.json");
|
|
166
|
-
if (
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
181
|
+
if (fs.existsSync(fopsConfig)) {
|
|
182
|
+
const raw = JSON.parse(fs.readFileSync(fopsConfig, "utf8"));
|
|
183
|
+
const entry = raw?.plugins?.entries?.[pluginId];
|
|
184
|
+
if (entry?.enabled === false) return false;
|
|
185
|
+
if (entry?.enabled === true) return true;
|
|
186
|
+
}
|
|
170
187
|
} catch {
|
|
171
188
|
// ignore parse errors
|
|
172
189
|
}
|
|
173
|
-
return
|
|
190
|
+
return !DEFAULT_DISABLED.has(pluginId);
|
|
174
191
|
}
|
|
175
192
|
|
|
176
193
|
/**
|