@galaxy-io/filament-ui 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/queries/pipeline_versions.d.ts +5 -0
- package/dist/components/connections/ConnectorPreviewChip.d.ts +7 -0
- package/dist/components/connections/ConnectorPreviewChip.js +11 -0
- package/dist/gen/ingestion/v1/connectors_pb.d.ts +11 -30
- package/dist/gen/ingestion/v1/connectors_pb.js +2 -24
- package/dist/gen/ingestion/v1/pipelines_pb.d.ts +29 -0
- package/dist/gen/ingestion/v1/pipelines_pb.js +17 -12
- package/dist/module/commands.js +1 -1
- package/dist/pages/connections/ConnectionsPage.js +5 -1
- package/dist/pages/connections/components/card/ConnectionCard.d.ts +1 -0
- package/dist/pages/connections/components/card/ConnectionCard.js +9 -4
- package/dist/pages/connections/components/create/select/CreateConnectionSelectorCard.js +9 -4
- package/dist/pages/connections/components/create/select/CreateConnectionSelectorSidebar.js +2 -2
- package/dist/pages/connections/components/create/select/hooks/useCreateConnectionSelectorCatalog.js +1 -1
- package/dist/pages/connections/components/create/types.d.ts +1 -3
- package/dist/pages/connections/components/create/types.js +1 -3
- package/dist/pages/connections/components/create/utils.js +2 -3
- package/dist/pages/connections/components/form/ConnectionForm.js +2 -3
- package/dist/pages/connections/components/form/ConnectionFormPreviewAlert.d.ts +10 -0
- package/dist/pages/connections/components/form/{ConnectionFormMaturityAlert.js → ConnectionFormPreviewAlert.js} +5 -8
- package/dist/pages/connections/constants.d.ts +0 -3
- package/dist/pages/connections/constants.js +3 -23
- package/dist/pages/pipelines/canvas/constants.d.ts +1 -0
- package/dist/pages/pipelines/canvas/constants.js +2 -1
- package/dist/pages/pipelines/canvas/graph/diff.js +4 -1
- package/dist/pages/pipelines/canvas/graph/serialize.js +1 -0
- package/dist/pages/pipelines/canvas/hooks/usePipelineCanvasEdgeResources.d.ts +2 -1
- package/dist/pages/pipelines/canvas/hooks/usePipelineCanvasEdgeResources.js +2 -0
- package/dist/pages/pipelines/canvas/hooks/usePipelineCanvasPanelResourceOptions.d.ts +1 -0
- package/dist/pages/pipelines/canvas/hooks/usePipelineCanvasPanelResourceOptions.js +2 -1
- package/dist/pages/pipelines/canvas/panel/overview/resource/PipelineCanvasPanelResourceDetail.js +29 -5
- package/dist/pages/pipelines/canvas/panel/overview/resource/PipelineCanvasPanelResourcePartitionsSection.d.ts +11 -0
- package/dist/pages/pipelines/canvas/panel/overview/resource/PipelineCanvasPanelResourcePartitionsSection.js +35 -0
- package/dist/pages/pipelines/canvas/routes/PipelineCanvasRoutesRowControls.js +10 -3
- package/dist/pages/pipelines/canvas/types.d.ts +1 -0
- package/dist/pages/pipelines/canvas/utils.d.ts +2 -1
- package/dist/pages/pipelines/canvas/utils.js +2 -1
- package/dist/pages/pipelines/components/create/CreatePipelineModalProvider.d.ts +2 -1
- package/dist/pages/pipelines/components/create/CreatePipelineModalProvider.js +4 -0
- package/dist/pages/pipelines/components/create/actions.d.ts +11 -2
- package/dist/pages/pipelines/components/create/actions.js +1 -0
- package/dist/pages/pipelines/components/create/hooks/useCreatePipelineResources.js +9 -3
- package/dist/pages/pipelines/components/create/reducer.js +16 -0
- package/dist/pages/pipelines/components/create/rows.js +9 -1
- package/dist/pages/pipelines/components/create/serialize.js +8 -2
- package/dist/pages/pipelines/components/create/steps/components/CreatePipelineModalResourcesColumnsPane.js +90 -31
- package/dist/pages/pipelines/components/create/steps/components/CreatePipelineModalResourcesPaneSection.d.ts +9 -0
- package/dist/pages/pipelines/components/create/steps/components/CreatePipelineModalResourcesPaneSection.js +78 -0
- package/dist/pages/pipelines/components/create/types.d.ts +4 -1
- package/dist/pages/pipelines/components/create/utils.js +1 -0
- package/dist/pages/pipelines/components/header/PipelinePageSaveIssues.js +12 -9
- package/dist/pages/pipelines/components/resource/PipelineResourcePartitionsChip.d.ts +7 -0
- package/dist/pages/pipelines/components/resource/PipelineResourcePartitionsChip.js +11 -0
- package/dist/pages/pipelines/components/resource/PipelineResourcePartitionsTable.d.ts +11 -0
- package/dist/pages/pipelines/components/resource/PipelineResourcePartitionsTable.js +45 -0
- package/dist/pages/pipelines/components/resource/types.d.ts +6 -2
- package/dist/pages/pipelines/components/resource/types.js +1 -0
- package/dist/pages/pipelines/components/resource/utils.d.ts +11 -2
- package/dist/pages/pipelines/components/resource/utils.js +7 -2
- package/dist/styles.css +3 -0
- package/package.json +3 -3
- package/dist/pages/connections/components/form/ConnectionFormMaturityAlert.d.ts +0 -11
|
@@ -58,6 +58,7 @@ var PIPELINE_CANVAS_NODE_TYPE_TO_COUNTERPART_TYPE_MAP = {
|
|
|
58
58
|
[PipelineCanvasNodeType.SINK]: PipelineCanvasNodeType.SOURCE
|
|
59
59
|
};
|
|
60
60
|
var PIPELINE_CANVAS_EDGE_TYPE = "pipeline";
|
|
61
|
+
var PIPELINE_CANVAS_EDGE_WRITE_MODE_FIELD = "write_mode";
|
|
61
62
|
var PIPELINE_CANVAS_EDGE_Z_INDEX = 2e3;
|
|
62
63
|
var PIPELINE_CANVAS_VIEW_TO_LABEL_MAP = {
|
|
63
64
|
[PipelineCanvasView.CANVAS]: "Canvas",
|
|
@@ -68,4 +69,4 @@ var PIPELINE_CANVAS_VIEW_TO_ICON_MAP = {
|
|
|
68
69
|
[PipelineCanvasView.ROUTES]: FlowArrowIcon
|
|
69
70
|
};
|
|
70
71
|
//#endregion
|
|
71
|
-
export { CONNECTOR_KIND_TO_HANDLE_ID_MAP, CONNECTOR_KIND_TO_HANDLE_TYPE_MAP, CONNECTOR_KIND_TO_NODE_TYPE_MAP, CONNECTOR_KIND_TO_NORMALIZED_KIND_MAP, CONNECTOR_KIND_TO_XYFLOW_POSITION_MAP, PIPELINE_CANVAS_EDGE_TYPE, PIPELINE_CANVAS_EDGE_Z_INDEX, PIPELINE_CANVAS_EDIT_MODE_TO_ICON_MAP, PIPELINE_CANVAS_FIT_MIN_ZOOM, PIPELINE_CANVAS_FIT_PADDING, PIPELINE_CANVAS_NODE_SINK_HANDLE_ID, PIPELINE_CANVAS_NODE_SOURCE_HANDLE_ID, PIPELINE_CANVAS_NODE_TYPE_TO_CONNECTOR_KIND_MAP, PIPELINE_CANVAS_NODE_TYPE_TO_COUNTERPART_TYPE_MAP, PIPELINE_CANVAS_NO_RUNS, PIPELINE_CANVAS_PAN_ON_DRAG, PIPELINE_CANVAS_REVEAL_ZOOM, PIPELINE_CANVAS_SNAP_GRID, PIPELINE_CANVAS_SOURCE_NODE_EXPECTED_HEIGHT, PIPELINE_CANVAS_VIEW_TO_ICON_MAP, PIPELINE_CANVAS_VIEW_TO_LABEL_MAP };
|
|
72
|
+
export { CONNECTOR_KIND_TO_HANDLE_ID_MAP, CONNECTOR_KIND_TO_HANDLE_TYPE_MAP, CONNECTOR_KIND_TO_NODE_TYPE_MAP, CONNECTOR_KIND_TO_NORMALIZED_KIND_MAP, CONNECTOR_KIND_TO_XYFLOW_POSITION_MAP, PIPELINE_CANVAS_EDGE_TYPE, PIPELINE_CANVAS_EDGE_WRITE_MODE_FIELD, PIPELINE_CANVAS_EDGE_Z_INDEX, PIPELINE_CANVAS_EDIT_MODE_TO_ICON_MAP, PIPELINE_CANVAS_FIT_MIN_ZOOM, PIPELINE_CANVAS_FIT_PADDING, PIPELINE_CANVAS_NODE_SINK_HANDLE_ID, PIPELINE_CANVAS_NODE_SOURCE_HANDLE_ID, PIPELINE_CANVAS_NODE_TYPE_TO_CONNECTOR_KIND_MAP, PIPELINE_CANVAS_NODE_TYPE_TO_COUNTERPART_TYPE_MAP, PIPELINE_CANVAS_NO_RUNS, PIPELINE_CANVAS_PAN_ON_DRAG, PIPELINE_CANVAS_REVEAL_ZOOM, PIPELINE_CANVAS_SNAP_GRID, PIPELINE_CANVAS_SOURCE_NODE_EXPECTED_HEIGHT, PIPELINE_CANVAS_VIEW_TO_ICON_MAP, PIPELINE_CANVAS_VIEW_TO_LABEL_MAP };
|
|
@@ -8,9 +8,12 @@ var getCanonicalJson = (value) => {
|
|
|
8
8
|
return value;
|
|
9
9
|
};
|
|
10
10
|
var getNodeConfigKey = (config) => config && Object.keys(config).length > 0 ? JSON.stringify(getCanonicalJson(config)) : "";
|
|
11
|
-
var getEdgeConfigKey = ({ readMode, writeMode, cursors, columns, transform, destinationResource }) => `${JSON.stringify(destinationResource ?? "")}|${readMode}|${writeMode}|${cursors.map((cursor) => `${cursor.resource}:${cursor.field}:${cursor.lookbackSeconds}`).sort().join(";")}|${JSON.stringify((columns ?? []).map((selection) => ({
|
|
11
|
+
var getEdgeConfigKey = ({ readMode, writeMode, cursors, columns, partitions, transform, destinationResource }) => `${JSON.stringify(destinationResource ?? "")}|${readMode}|${writeMode}|${cursors.map((cursor) => `${cursor.resource}:${cursor.field}:${cursor.lookbackSeconds}`).sort().join(";")}|${JSON.stringify((columns ?? []).map((selection) => ({
|
|
12
12
|
resource: selection.resource,
|
|
13
13
|
columns: [...selection.columns].sort()
|
|
14
|
+
})).sort((left, right) => left.resource.localeCompare(right.resource)))}|${JSON.stringify((partitions ?? []).map((selection) => ({
|
|
15
|
+
resource: selection.resource,
|
|
16
|
+
partitions: [...selection.partitions].sort()
|
|
14
17
|
})).sort((left, right) => left.resource.localeCompare(right.resource)))}|${transform ? JSON.stringify(getCanonicalJson(transform)) : ""}`;
|
|
15
18
|
var hasPipelineGraphChanges = (state, version) => {
|
|
16
19
|
const versionNodesById = new Map((version?.graph?.nodes ?? []).map((node) => [node.id, node]));
|
|
@@ -12,6 +12,7 @@ var getCanvasEdgeConfig = (edge, baseEdge) => ({
|
|
|
12
12
|
writeMode: edge.data?.writeMode ?? baseEdge?.writeMode ?? WriteMode.UNSPECIFIED,
|
|
13
13
|
cursors: edge.data?.cursors ?? baseEdge?.cursors ?? [],
|
|
14
14
|
columns: edge.data?.columns ?? baseEdge?.columns ?? [],
|
|
15
|
+
partitions: edge.data?.partitions ?? baseEdge?.partitions ?? [],
|
|
15
16
|
transform: edge.data && "transform" in edge.data ? edge.data.transform : baseEdge?.transform
|
|
16
17
|
});
|
|
17
18
|
var getProtoEdgeKey = (edge) => `${edge.fromNode}|${edge.resource}|${edge.toNode}`;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Connection } from '../../../../gen/ingestion/v1/connections_pb';
|
|
2
|
-
import { Resource, ResourceColumn } from '../../../../gen/ingestion/v1/connectors_pb';
|
|
2
|
+
import { Resource, ResourceColumn, ResourceColumns } from '../../../../gen/ingestion/v1/connectors_pb';
|
|
3
3
|
import { CanvasEdge } from '../types';
|
|
4
4
|
interface PipelineCanvasEdgeResourcesOptions {
|
|
5
5
|
enabled?: boolean;
|
|
@@ -16,6 +16,7 @@ export interface PipelineCanvasEdgeResources {
|
|
|
16
16
|
defaultCursor: ResourceColumn["name"];
|
|
17
17
|
isManagedIncremental: boolean;
|
|
18
18
|
isColumnSelectable: boolean;
|
|
19
|
+
partitions: ResourceColumns["partitions"];
|
|
19
20
|
hasPrimaryKey: boolean;
|
|
20
21
|
}
|
|
21
22
|
export declare const usePipelineCanvasEdgeResources: (edge: CanvasEdge, { enabled }?: PipelineCanvasEdgeResourcesOptions) => PipelineCanvasEdgeResources;
|
|
@@ -9,6 +9,7 @@ import { usePipelineExecutionMode } from "../../hooks/usePipelineExecutionMode.j
|
|
|
9
9
|
import { useMemo } from "react";
|
|
10
10
|
import { create } from "@bufbuild/protobuf";
|
|
11
11
|
//#region src/pages/pipelines/canvas/hooks/usePipelineCanvasEdgeResources.ts
|
|
12
|
+
var NO_PARTITIONS = [];
|
|
12
13
|
var usePipelineCanvasEdgeResources = (edge, { enabled = true } = {}) => {
|
|
13
14
|
const isContinuous = usePipelineExecutionMode() === ExecutionMode.CONTINUOUS;
|
|
14
15
|
const sourceConnection = usePipelineCanvasConnections().get(edge.source);
|
|
@@ -50,6 +51,7 @@ var usePipelineCanvasEdgeResources = (edge, { enabled = true } = {}) => {
|
|
|
50
51
|
defaultCursor: isManagedIncremental ? "" : getDefaultCursor(columns ?? []),
|
|
51
52
|
isManagedIncremental,
|
|
52
53
|
isColumnSelectable: entry?.selectableColumns ?? false,
|
|
54
|
+
partitions: entry?.partitions ?? NO_PARTITIONS,
|
|
53
55
|
hasPrimaryKey: (discovered?.resources.find((candidate) => candidate.name === resource)?.primaryKey.length ?? 0) > 0
|
|
54
56
|
};
|
|
55
57
|
};
|
|
@@ -18,5 +18,6 @@ export declare const usePipelineCanvasPanelResourceOptions: (edge: CanvasEdge) =
|
|
|
18
18
|
defaultCursor: import('../../../../gen/ingestion/v1/connectors_pb').ResourceColumn["name"];
|
|
19
19
|
isManagedIncremental: boolean;
|
|
20
20
|
isColumnSelectable: boolean;
|
|
21
|
+
partitions: import('../../../../gen/ingestion/v1/connectors_pb').ResourceColumns["partitions"];
|
|
21
22
|
hasPrimaryKey: boolean;
|
|
22
23
|
};
|
|
@@ -47,7 +47,8 @@ var usePipelineCanvasPanelResourceOptions = (edge) => {
|
|
|
47
47
|
readMode: edge.data?.readMode ?? ReadMode.UNSPECIFIED,
|
|
48
48
|
writeMode: edge.data?.writeMode ?? WriteMode.UNSPECIFIED,
|
|
49
49
|
cursors: edge.data?.cursors ?? [],
|
|
50
|
-
columns: edge.data?.columns ?? []
|
|
50
|
+
columns: edge.data?.columns ?? [],
|
|
51
|
+
partitions: edge.data?.partitions ?? []
|
|
51
52
|
})]
|
|
52
53
|
})
|
|
53
54
|
}), [
|
package/dist/pages/pipelines/canvas/panel/overview/resource/PipelineCanvasPanelResourceDetail.js
CHANGED
|
@@ -3,8 +3,9 @@ import { getEnumSelectValue, mapOptionIdToEnum } from "../../../../../../utils/s
|
|
|
3
3
|
import KeyValueList from "../../../../../../components/KeyValueList.js";
|
|
4
4
|
import KeyValueListRow from "../../../../../../components/KeyValueListRow.js";
|
|
5
5
|
import { PipelineResourceStatusField } from "../../../../components/resource/types.js";
|
|
6
|
-
import { createResourceColumnSelections, getEffectiveColumnSelection, getLockedColumnNames } from "../../../../components/resource/utils.js";
|
|
7
|
-
import
|
|
6
|
+
import { createResourceColumnSelections, createResourcePartitionSelections, getEffectiveColumnSelection, getEffectivePartitionSelection, getLockedColumnNames } from "../../../../components/resource/utils.js";
|
|
7
|
+
import "../../../constants.js";
|
|
8
|
+
import { getDefaultDestinationResource, getEdgeFieldError, getEdgeResourceStatus } from "../../../utils.js";
|
|
8
9
|
import { usePipelineCanvasActions, usePipelineCanvasReadOnly } from "../../../providers/canvas/PipelineCanvasProvider.js";
|
|
9
10
|
import { usePipelineCanvasSelection } from "../../../hooks/usePipelineCanvasSelection.js";
|
|
10
11
|
import PipelineCanvasPanelSection from "../../PipelineCanvasPanelSection.js";
|
|
@@ -14,6 +15,7 @@ import { usePipelineCanvasPanelResourceOptions } from "../../../hooks/usePipelin
|
|
|
14
15
|
import PipelineCanvasPanelResourceColumnsSection from "./PipelineCanvasPanelResourceColumnsSection.js";
|
|
15
16
|
import PipelineCanvasPanelResourceCursorField from "./PipelineCanvasPanelResourceCursorField.js";
|
|
16
17
|
import PipelineCanvasPanelResourceEndpoint from "./PipelineCanvasPanelResourceEndpoint.js";
|
|
18
|
+
import PipelineCanvasPanelResourcePartitionsSection from "./PipelineCanvasPanelResourcePartitionsSection.js";
|
|
17
19
|
import PipelineTransformFieldsProvider from "../../../../components/transform/PipelineTransformFieldsProvider.js";
|
|
18
20
|
import PipelineCanvasPanelResourceTransformSection from "./transform/PipelineCanvasPanelResourceTransformSection.js";
|
|
19
21
|
import PipelineCanvasPanelResourceTransformSectionError from "./transform/PipelineCanvasPanelResourceTransformSectionError.js";
|
|
@@ -38,7 +40,7 @@ var PipelineCanvasPanelResourceDetail = ({ edge }) => {
|
|
|
38
40
|
const { clearSelection, setShowPanel } = usePipelineCanvasSelection();
|
|
39
41
|
const { setEdgeConfig, applyEdgeChanges } = usePipelineCanvasActions();
|
|
40
42
|
const options = usePipelineCanvasPanelResourceOptions(edge);
|
|
41
|
-
const { isContinuous, hasReadLevers, isTransformable, isLoading, isLoadingColumns, sourceConnectionId, resource, columns, verdict, readModeOptions, writeModeOptions, effectiveReadMode, effectiveWriteMode, cursorOptions, defaultCursor, isManagedIncremental, isColumnSelectable } = options;
|
|
43
|
+
const { isContinuous, hasReadLevers, isTransformable, isLoading, isLoadingColumns, sourceConnectionId, resource, columns, verdict, readModeOptions, writeModeOptions, effectiveReadMode, effectiveWriteMode, cursorOptions, defaultCursor, isManagedIncremental, isColumnSelectable, partitions } = options;
|
|
42
44
|
const { configuredReadMode, configuredWriteMode, readMode, writeMode, cursors, cursor, readModeSelectOptions, writeModeSelectOptions, handleReadModeChange, handleWriteModeChange, handleCursorChange } = usePipelineCanvasEdgeConfig(edge, {
|
|
43
45
|
readModeOptions,
|
|
44
46
|
writeModeOptions,
|
|
@@ -91,7 +93,7 @@ var PipelineCanvasPanelResourceDetail = ({ edge }) => {
|
|
|
91
93
|
lockedColumnNames
|
|
92
94
|
]);
|
|
93
95
|
const selectedColumns = useMemo(() => (columns ?? NO_COLUMNS).filter((column) => selectedColumnNames.includes(column.name)), [columns, selectedColumnNames]);
|
|
94
|
-
const columnsError = verdict
|
|
96
|
+
const columnsError = getEdgeFieldError(verdict, PipelineResourceStatusField.COLUMNS);
|
|
95
97
|
const handleColumnsChange = (next) => setEdgeConfig(edge.id, {
|
|
96
98
|
readMode: configuredReadMode,
|
|
97
99
|
writeMode: configuredWriteMode,
|
|
@@ -102,6 +104,21 @@ var PipelineCanvasPanelResourceDetail = ({ edge }) => {
|
|
|
102
104
|
columns: columns ?? NO_COLUMNS
|
|
103
105
|
})
|
|
104
106
|
});
|
|
107
|
+
const selectedPartitions = getEffectivePartitionSelection({
|
|
108
|
+
stored: edge.data?.partitions?.find((candidate) => candidate.resource === resource)?.partitions,
|
|
109
|
+
partitions
|
|
110
|
+
});
|
|
111
|
+
const partitionsError = getEdgeFieldError(verdict, PipelineResourceStatusField.PARTITIONS);
|
|
112
|
+
const handlePartitionsChange = (next) => setEdgeConfig(edge.id, {
|
|
113
|
+
readMode: configuredReadMode,
|
|
114
|
+
writeMode: configuredWriteMode,
|
|
115
|
+
cursors,
|
|
116
|
+
partitions: createResourcePartitionSelections({
|
|
117
|
+
resource,
|
|
118
|
+
selected: next,
|
|
119
|
+
partitions
|
|
120
|
+
})
|
|
121
|
+
});
|
|
105
122
|
return /* @__PURE__ */ jsxs(Fragment$1, { children: [/* @__PURE__ */ jsx(PipelineCanvasPanelHeader, {
|
|
106
123
|
title: resource,
|
|
107
124
|
icon: FlowArrowIcon,
|
|
@@ -194,7 +211,7 @@ var PipelineCanvasPanelResourceDetail = ({ edge }) => {
|
|
|
194
211
|
},
|
|
195
212
|
placeholder: "Select a write mode...",
|
|
196
213
|
isDisabled: isReadOnly || isLoading,
|
|
197
|
-
error: getFieldError(PipelineResourceStatusField.WRITE_MODE),
|
|
214
|
+
error: getFieldError(PipelineResourceStatusField.WRITE_MODE) ?? getEdgeFieldError(verdict, "write_mode"),
|
|
198
215
|
variant: SelectInputVariant.TERTIARY,
|
|
199
216
|
fillWidth: true
|
|
200
217
|
}),
|
|
@@ -218,6 +235,13 @@ var PipelineCanvasPanelResourceDetail = ({ edge }) => {
|
|
|
218
235
|
error: columnsError,
|
|
219
236
|
onChange: handleColumnsChange
|
|
220
237
|
}),
|
|
238
|
+
partitions.length > 0 && /* @__PURE__ */ jsx(PipelineCanvasPanelResourcePartitionsSection, {
|
|
239
|
+
partitions,
|
|
240
|
+
selected: selectedPartitions,
|
|
241
|
+
isDisabled: isReadOnly || isLoading,
|
|
242
|
+
error: partitionsError,
|
|
243
|
+
onChange: handlePartitionsChange
|
|
244
|
+
}),
|
|
221
245
|
isTransformable && (isLoadingColumns ? /* @__PURE__ */ jsx(PipelineCanvasPanelResourceTransformSectionPending, {}) : /* @__PURE__ */ jsx(CatchBoundary, {
|
|
222
246
|
getResetKey: () => edge.id,
|
|
223
247
|
errorComponent: PipelineCanvasPanelResourceTransformSectionError,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { FC } from 'react';
|
|
2
|
+
import { ResourceColumns } from '../../../../../../gen/ingestion/v1/connectors_pb';
|
|
3
|
+
interface PipelineCanvasPanelResourcePartitionsSectionProps {
|
|
4
|
+
partitions: ResourceColumns["partitions"];
|
|
5
|
+
selected: ResourceColumns["partitions"];
|
|
6
|
+
isDisabled: boolean;
|
|
7
|
+
error?: string;
|
|
8
|
+
onChange: (selected: ResourceColumns["partitions"]) => void;
|
|
9
|
+
}
|
|
10
|
+
declare const PipelineCanvasPanelResourcePartitionsSection: FC<PipelineCanvasPanelResourcePartitionsSectionProps>;
|
|
11
|
+
export default PipelineCanvasPanelResourcePartitionsSection;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import PipelineResourcePartitionsChip from "../../../../components/resource/PipelineResourcePartitionsChip.js";
|
|
2
|
+
import PipelineResourcePartitionsTable from "../../../../components/resource/PipelineResourcePartitionsTable.js";
|
|
3
|
+
import PipelineCanvasPanelSection from "../../PipelineCanvasPanelSection.js";
|
|
4
|
+
import { SquaresFourIcon } from "@phosphor-icons/react";
|
|
5
|
+
import { jsx, jsxs } from "react/jsx-runtime";
|
|
6
|
+
import Box from "@galaxy-io/dls/layout/Box";
|
|
7
|
+
import Text, { TextSize, TextVariant } from "@galaxy-io/dls/text/Text";
|
|
8
|
+
//#region src/pages/pipelines/canvas/panel/overview/resource/PipelineCanvasPanelResourcePartitionsSection.tsx
|
|
9
|
+
var PipelineCanvasPanelResourcePartitionsSection = ({ partitions, selected, isDisabled, error, onChange }) => /* @__PURE__ */ jsxs(PipelineCanvasPanelSection, {
|
|
10
|
+
icon: SquaresFourIcon,
|
|
11
|
+
header: "Partitions",
|
|
12
|
+
isEmpty: partitions.length === 0,
|
|
13
|
+
emptyHeader: "No partitions",
|
|
14
|
+
emptyMessage: "This resource is not partitioned.",
|
|
15
|
+
isOpenInitial: false,
|
|
16
|
+
trailing: /* @__PURE__ */ jsx(PipelineResourcePartitionsChip, {
|
|
17
|
+
selectedCount: selected.length,
|
|
18
|
+
totalCount: partitions.length
|
|
19
|
+
}),
|
|
20
|
+
children: [error && /* @__PURE__ */ jsx(Box, {
|
|
21
|
+
padding: 12,
|
|
22
|
+
children: /* @__PURE__ */ jsx(Text, {
|
|
23
|
+
size: TextSize.BODY_SM,
|
|
24
|
+
variant: TextVariant.ERROR,
|
|
25
|
+
children: error
|
|
26
|
+
})
|
|
27
|
+
}), /* @__PURE__ */ jsx(PipelineResourcePartitionsTable, {
|
|
28
|
+
partitions,
|
|
29
|
+
selected,
|
|
30
|
+
isDisabled,
|
|
31
|
+
onChange
|
|
32
|
+
})]
|
|
33
|
+
});
|
|
34
|
+
//#endregion
|
|
35
|
+
export { PipelineCanvasPanelResourcePartitionsSection as default };
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { ReadMode, WriteMode } from "../../../../gen/ingestion/v1/common_pb.js";
|
|
2
2
|
import { getEnumSelectValue, mapOptionIdToEnum } from "../../../../utils/select.js";
|
|
3
3
|
import { getCursorSelectOptions } from "../../components/resource/utils.js";
|
|
4
|
-
import {
|
|
4
|
+
import { PIPELINE_CANVAS_EDGE_WRITE_MODE_FIELD } from "../constants.js";
|
|
5
|
+
import { getEdgeFieldError, getEdgeModeOptions, getEdgeResourceStatus } from "../utils.js";
|
|
5
6
|
import { usePipelineCanvasReadOnly } from "../providers/canvas/PipelineCanvasProvider.js";
|
|
6
7
|
import { usePipelineCanvasEdgeConfig } from "../hooks/usePipelineCanvasEdgeConfig.js";
|
|
7
8
|
import { usePipelineCanvasEdgeResources } from "../hooks/usePipelineCanvasEdgeResources.js";
|
|
@@ -41,7 +42,12 @@ var PipelineCanvasRoutesRowControls = ({ route, children }) => {
|
|
|
41
42
|
writeMode,
|
|
42
43
|
cursor
|
|
43
44
|
});
|
|
44
|
-
const
|
|
45
|
+
const writeModeError = getEdgeFieldError(route.verdict, PIPELINE_CANVAS_EDGE_WRITE_MODE_FIELD);
|
|
46
|
+
const issues = [
|
|
47
|
+
...route.issues,
|
|
48
|
+
...status?.isBlocking ? [status.message] : [],
|
|
49
|
+
...writeModeError !== void 0 ? [writeModeError] : []
|
|
50
|
+
];
|
|
45
51
|
const warnings = status && !status.isBlocking ? [status.message] : [];
|
|
46
52
|
return /* @__PURE__ */ jsxs(Fragment, { children: [
|
|
47
53
|
/* @__PURE__ */ jsxs(ControlsGroup, {
|
|
@@ -101,7 +107,8 @@ var PipelineCanvasRoutesRowControls = ({ route, children }) => {
|
|
|
101
107
|
variant: SelectInputVariant.SECONDARY,
|
|
102
108
|
size: SelectInputSize.SMALL,
|
|
103
109
|
placeholder: "Write mode...",
|
|
104
|
-
isDisabled: isReadOnly
|
|
110
|
+
isDisabled: isReadOnly,
|
|
111
|
+
isError: writeModeError !== void 0
|
|
105
112
|
})
|
|
106
113
|
})
|
|
107
114
|
})
|
|
@@ -32,6 +32,7 @@ export type CanvasNode = PipelineCanvasSourceNode | PipelineCanvasSinkNode | Pip
|
|
|
32
32
|
export type PipelineCanvasEdgeTransform = JsonObject;
|
|
33
33
|
export type PipelineCanvasEdgeData = Pick<PipelineEdge, "readMode" | "writeMode" | "cursors"> & {
|
|
34
34
|
columns?: PipelineEdge["columns"];
|
|
35
|
+
partitions?: PipelineEdge["partitions"];
|
|
35
36
|
transform?: PipelineCanvasEdgeTransform;
|
|
36
37
|
destinationResource?: string;
|
|
37
38
|
};
|
|
@@ -3,7 +3,7 @@ import { Theme } from '@galaxy-io/dls/theme/tokens/types';
|
|
|
3
3
|
import { EdgeValidation } from '../../../gen/ingestion/v1/capabilities_pb';
|
|
4
4
|
import { ReadMode, WriteMode } from '../../../gen/ingestion/v1/common_pb';
|
|
5
5
|
import { Connection } from '../../../gen/ingestion/v1/connections_pb';
|
|
6
|
-
import { ResourceColumn } from '../../../gen/ingestion/v1/connectors_pb';
|
|
6
|
+
import { ResourceColumn, ValidationError } from '../../../gen/ingestion/v1/connectors_pb';
|
|
7
7
|
import { PipelineEdge } from '../../../gen/ingestion/v1/pipelines_pb';
|
|
8
8
|
import { RunInfo } from '../../../gen/ingestion/v1/runs_pb';
|
|
9
9
|
import { PipelineFlowEndpoints } from '../../../components/pipelines/utils';
|
|
@@ -17,6 +17,7 @@ export interface PipelineCanvasEdgeModeOptions {
|
|
|
17
17
|
effectiveWriteMode: WriteMode;
|
|
18
18
|
}
|
|
19
19
|
export declare const getEdgeModeOptions: (verdict: EdgeValidation | undefined, hasReadLevers: boolean) => PipelineCanvasEdgeModeOptions;
|
|
20
|
+
export declare const getEdgeFieldError: (verdict: EdgeValidation | undefined, field: ValidationError["field"]) => ValidationError["message"] | undefined;
|
|
20
21
|
type PipelineCanvasEdgeStatusResources = Pick<PipelineCanvasEdgeResources, "hasReadLevers" | "resource" | "columns" | "cursorOptions" | "isManagedIncremental" | "hasPrimaryKey">;
|
|
21
22
|
interface PipelineCanvasEdgeStatusConfig {
|
|
22
23
|
verdict: EdgeValidation | undefined;
|
|
@@ -10,6 +10,7 @@ var getEdgeModeOptions = (verdict, hasReadLevers) => ({
|
|
|
10
10
|
effectiveReadMode: verdict?.effectiveReadMode ?? ReadMode.UNSPECIFIED,
|
|
11
11
|
effectiveWriteMode: verdict?.effectiveWriteMode ?? WriteMode.UNSPECIFIED
|
|
12
12
|
});
|
|
13
|
+
var getEdgeFieldError = (verdict, field) => verdict?.errors.find((error) => error.field === field)?.message;
|
|
13
14
|
var getEdgeResourceStatus = ({ hasReadLevers, resource, columns, cursorOptions, isManagedIncremental, hasPrimaryKey }, { verdict, readMode, writeMode, cursor }) => {
|
|
14
15
|
if (!hasReadLevers || verdict === void 0 || resource === "") return void 0;
|
|
15
16
|
return getPipelineResourceStatus({
|
|
@@ -74,4 +75,4 @@ var mapCanvasNodesToFlowEndpoints = (nodes) => ({
|
|
|
74
75
|
var isConnectionNode = (node) => node.type === PipelineCanvasNodeType.SOURCE || node.type === PipelineCanvasNodeType.SINK;
|
|
75
76
|
var isPipelineCanvasConnectionDisabled = (connection, isSourceDisabled) => isSourceDisabled && connection.kind === ConnectorKind.SOURCE;
|
|
76
77
|
//#endregion
|
|
77
|
-
export { getDefaultDestinationResource, getEdgeModeOptions, getEdgeResourceStatus, getPipelineCanvasRevealBounds, getTransformStepCount, isConnectionNode, isPipelineCanvasConnectionDisabled, isPipelineCanvasRouteRunning, mapCanvasNodesToFlowEndpoints, mapEdgesToStyledEdges, mapElementsToSelected };
|
|
78
|
+
export { getDefaultDestinationResource, getEdgeFieldError, getEdgeModeOptions, getEdgeResourceStatus, getPipelineCanvasRevealBounds, getTransformStepCount, isConnectionNode, isPipelineCanvasConnectionDisabled, isPipelineCanvasRouteRunning, mapCanvasNodesToFlowEndpoints, mapEdgesToStyledEdges, mapElementsToSelected };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { FC, PropsWithChildren } from 'react';
|
|
2
|
-
import { AddNotifierAction, AddResourceAction, GoToStepAction, RemoveNotifierAction, SelectSourceAction, SetActiveSinkAction, SetDescriptionAction, SetExecutionModeAction, SetNameAction, SetNodeConfigAction, SetResourceColumnsAction, SetResourceCursorAction, SetResourceReadModeAction, SetResourceSelectionAction, SetResourceWriteModeAction, SetScheduleAction, SetSinkWriteModeAction, SetSubmittingAction, SetWorkerConfigurationAction, ToggleSinkAction, UpdateNotifierAction } from './actions';
|
|
2
|
+
import { AddNotifierAction, AddResourceAction, GoToStepAction, RemoveNotifierAction, SelectSourceAction, SetActiveSinkAction, SetDescriptionAction, SetExecutionModeAction, SetNameAction, SetNodeConfigAction, SetResourceColumnsAction, SetResourceCursorAction, SetResourcePartitionsAction, SetResourceReadModeAction, SetResourceSelectionAction, SetResourceWriteModeAction, SetScheduleAction, SetSinkWriteModeAction, SetSubmittingAction, SetWorkerConfigurationAction, ToggleSinkAction, UpdateNotifierAction } from './actions';
|
|
3
3
|
import { CreatePipelineModalContextValue } from './types';
|
|
4
4
|
export declare const useCreatePipelineModalState: () => CreatePipelineModalContextValue;
|
|
5
5
|
export declare const useCreatePipelineModalActions: () => {
|
|
@@ -12,6 +12,7 @@ export declare const useCreatePipelineModalActions: () => {
|
|
|
12
12
|
setResourceReadMode: (payload: SetResourceReadModeAction["payload"]) => void;
|
|
13
13
|
setResourceCursor: (payload: SetResourceCursorAction["payload"]) => void;
|
|
14
14
|
setResourceColumns: (payload: SetResourceColumnsAction["payload"]) => void;
|
|
15
|
+
setResourcePartitions: (payload: SetResourcePartitionsAction["payload"]) => void;
|
|
15
16
|
setResourceWriteMode: (payload: SetResourceWriteModeAction["payload"]) => void;
|
|
16
17
|
setSinkWriteMode: (payload: SetSinkWriteModeAction["payload"]) => void;
|
|
17
18
|
setNodeConfig: (payload: SetNodeConfigAction["payload"]) => void;
|
|
@@ -68,6 +68,10 @@ var useCreatePipelineModalActions = () => {
|
|
|
68
68
|
type: CreatePipelineModalActionType.SET_RESOURCE_COLUMNS,
|
|
69
69
|
payload
|
|
70
70
|
}),
|
|
71
|
+
setResourcePartitions: (payload) => dispatch({
|
|
72
|
+
type: CreatePipelineModalActionType.SET_RESOURCE_PARTITIONS,
|
|
73
|
+
payload
|
|
74
|
+
}),
|
|
71
75
|
setResourceWriteMode: (payload) => dispatch({
|
|
72
76
|
type: CreatePipelineModalActionType.SET_RESOURCE_WRITE_MODE,
|
|
73
77
|
payload
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ExecutionMode, ReadMode, WriteMode } from '../../../../gen/ingestion/v1/common_pb';
|
|
2
2
|
import { Connection } from '../../../../gen/ingestion/v1/connections_pb';
|
|
3
|
-
import { Resource, ResourceColumn } from '../../../../gen/ingestion/v1/connectors_pb';
|
|
3
|
+
import { Resource, ResourceColumn, ResourceColumns } from '../../../../gen/ingestion/v1/connectors_pb';
|
|
4
4
|
import { Pipeline } from '../../../../gen/ingestion/v1/pipelines_pb';
|
|
5
5
|
import { CreatePipelineModalStep } from './types';
|
|
6
6
|
import { PipelineNodeConfig } from '../node/types';
|
|
@@ -16,6 +16,7 @@ export declare enum CreatePipelineModalActionType {
|
|
|
16
16
|
SET_RESOURCE_READ_MODE = "SET_RESOURCE_READ_MODE",
|
|
17
17
|
SET_RESOURCE_CURSOR = "SET_RESOURCE_CURSOR",
|
|
18
18
|
SET_RESOURCE_COLUMNS = "SET_RESOURCE_COLUMNS",
|
|
19
|
+
SET_RESOURCE_PARTITIONS = "SET_RESOURCE_PARTITIONS",
|
|
19
20
|
SET_RESOURCE_WRITE_MODE = "SET_RESOURCE_WRITE_MODE",
|
|
20
21
|
SET_SINK_WRITE_MODE = "SET_SINK_WRITE_MODE",
|
|
21
22
|
SET_NODE_CONFIG = "SET_NODE_CONFIG",
|
|
@@ -86,6 +87,14 @@ export interface SetResourceColumnsAction {
|
|
|
86
87
|
columns: ResourceColumn["name"][];
|
|
87
88
|
};
|
|
88
89
|
}
|
|
90
|
+
export interface SetResourcePartitionsAction {
|
|
91
|
+
type: CreatePipelineModalActionType.SET_RESOURCE_PARTITIONS;
|
|
92
|
+
payload: {
|
|
93
|
+
sinkId: Connection["id"];
|
|
94
|
+
resource: Resource["name"];
|
|
95
|
+
partitions: ResourceColumns["partitions"];
|
|
96
|
+
};
|
|
97
|
+
}
|
|
89
98
|
export interface SetResourceWriteModeAction {
|
|
90
99
|
type: CreatePipelineModalActionType.SET_RESOURCE_WRITE_MODE;
|
|
91
100
|
payload: {
|
|
@@ -153,5 +162,5 @@ export interface SetSubmittingAction {
|
|
|
153
162
|
type: CreatePipelineModalActionType.SET_SUBMITTING;
|
|
154
163
|
payload: boolean;
|
|
155
164
|
}
|
|
156
|
-
export type CreatePipelineModalAction = SelectSourceAction | ToggleSinkAction | SetActiveSinkAction | SetExecutionModeAction | AddResourceAction | SetResourceSelectionAction | SetResourceReadModeAction | SetResourceCursorAction | SetResourceColumnsAction | SetResourceWriteModeAction | SetSinkWriteModeAction | SetNodeConfigAction | SetNameAction | SetDescriptionAction | SetScheduleAction | AddNotifierAction | UpdateNotifierAction | RemoveNotifierAction | SetWorkerConfigurationAction | GoToStepAction | GoBackAction | GoNextAction | SetSubmittingAction;
|
|
165
|
+
export type CreatePipelineModalAction = SelectSourceAction | ToggleSinkAction | SetActiveSinkAction | SetExecutionModeAction | AddResourceAction | SetResourceSelectionAction | SetResourceReadModeAction | SetResourceCursorAction | SetResourceColumnsAction | SetResourcePartitionsAction | SetResourceWriteModeAction | SetSinkWriteModeAction | SetNodeConfigAction | SetNameAction | SetDescriptionAction | SetScheduleAction | AddNotifierAction | UpdateNotifierAction | RemoveNotifierAction | SetWorkerConfigurationAction | GoToStepAction | GoBackAction | GoNextAction | SetSubmittingAction;
|
|
157
166
|
export {};
|
|
@@ -9,6 +9,7 @@ var CreatePipelineModalActionType = /* @__PURE__ */ function(CreatePipelineModal
|
|
|
9
9
|
CreatePipelineModalActionType["SET_RESOURCE_READ_MODE"] = "SET_RESOURCE_READ_MODE";
|
|
10
10
|
CreatePipelineModalActionType["SET_RESOURCE_CURSOR"] = "SET_RESOURCE_CURSOR";
|
|
11
11
|
CreatePipelineModalActionType["SET_RESOURCE_COLUMNS"] = "SET_RESOURCE_COLUMNS";
|
|
12
|
+
CreatePipelineModalActionType["SET_RESOURCE_PARTITIONS"] = "SET_RESOURCE_PARTITIONS";
|
|
12
13
|
CreatePipelineModalActionType["SET_RESOURCE_WRITE_MODE"] = "SET_RESOURCE_WRITE_MODE";
|
|
13
14
|
CreatePipelineModalActionType["SET_SINK_WRITE_MODE"] = "SET_SINK_WRITE_MODE";
|
|
14
15
|
CreatePipelineModalActionType["SET_NODE_CONFIG"] = "SET_NODE_CONFIG";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ConnectorKind, ExecutionMode, ReadMode, ReplicationMode, WriteMode } from "../../../../../gen/ingestion/v1/common_pb.js";
|
|
2
2
|
import { DiscoverResourcesRequestSchema, GetResourceColumnsRequestSchema, ResourceSchema } from "../../../../../gen/ingestion/v1/connectors_pb.js";
|
|
3
|
-
import { PipelineEdgeSchema, PipelineGraphSchema, PipelineNodeSchema, ResourceColumnSelectionSchema } from "../../../../../gen/ingestion/v1/pipelines_pb.js";
|
|
3
|
+
import { PipelineEdgeSchema, PipelineGraphSchema, PipelineNodeSchema, ResourceColumnSelectionSchema, ResourcePartitionSelectionSchema } from "../../../../../gen/ingestion/v1/pipelines_pb.js";
|
|
4
4
|
import { ValidatePipelineRequestSchema } from "../../../../../gen/ingestion/v1/capabilities_pb.js";
|
|
5
5
|
import { useDiscoverResourcesQuery, useGetResourceColumnsQuery } from "../../../../../api/queries/connectors.js";
|
|
6
6
|
import { useValidatePipelineQuery } from "../../../../../api/queries/capabilities.js";
|
|
@@ -43,12 +43,16 @@ var useCreatePipelineResources = (state, isOpen) => {
|
|
|
43
43
|
enabled: isOpen && connectionId !== "" && resourceNames.length > 0
|
|
44
44
|
}
|
|
45
45
|
});
|
|
46
|
-
const { sinkConnections, sinkWriteModes, nodeConfigs, resourceSelection, resourceColumns, executionMode } = state;
|
|
46
|
+
const { sinkConnections, sinkWriteModes, nodeConfigs, resourceSelection, resourceColumns, resourcePartitions, executionMode } = state;
|
|
47
47
|
const validationInput = useMemo(() => {
|
|
48
48
|
const createColumnSelections = (sinkId, resource) => Object.entries(resourceColumns[sinkId] ?? {}).filter(([name]) => resource === "" || name === resource).map(([name, columns]) => create(ResourceColumnSelectionSchema, {
|
|
49
49
|
resource: name,
|
|
50
50
|
columns
|
|
51
51
|
}));
|
|
52
|
+
const createPartitionSelections = (sinkId, resource) => Object.entries(resourcePartitions[sinkId] ?? {}).filter(([name]) => resource === "" || name === resource).map(([name, partitions]) => create(ResourcePartitionSelectionSchema, {
|
|
53
|
+
resource: name,
|
|
54
|
+
partitions
|
|
55
|
+
}));
|
|
52
56
|
const createSinkEdges = (sink) => {
|
|
53
57
|
const writeMode = sinkWriteModes[sink.id] ?? (hasReadLevers ? CREATE_PIPELINE_MODAL_DEFAULT_WRITE_MODE : WriteMode.APPEND);
|
|
54
58
|
const selected = isContinuous ? resources.filter((resource) => isResourceSelected(resourceSelection[sink.id], resource)) : [];
|
|
@@ -58,7 +62,8 @@ var useCreatePipelineResources = (state, isOpen) => {
|
|
|
58
62
|
resource,
|
|
59
63
|
readMode: hasReadLevers ? ReadMode.FULL : ReadMode.UNSPECIFIED,
|
|
60
64
|
writeMode,
|
|
61
|
-
columns: createColumnSelections(sink.id, resource)
|
|
65
|
+
columns: createColumnSelections(sink.id, resource),
|
|
66
|
+
partitions: createPartitionSelections(sink.id, resource)
|
|
62
67
|
}));
|
|
63
68
|
};
|
|
64
69
|
return create(ValidatePipelineRequestSchema, {
|
|
@@ -85,6 +90,7 @@ var useCreatePipelineResources = (state, isOpen) => {
|
|
|
85
90
|
nodeConfigs,
|
|
86
91
|
resourceSelection,
|
|
87
92
|
resourceColumns,
|
|
93
|
+
resourcePartitions,
|
|
88
94
|
executionMode,
|
|
89
95
|
resources,
|
|
90
96
|
hasReadLevers,
|
|
@@ -13,6 +13,7 @@ function applyExecutionMode(state, executionMode) {
|
|
|
13
13
|
resourceReadModes: {},
|
|
14
14
|
resourceCursors: {},
|
|
15
15
|
resourceColumns: {},
|
|
16
|
+
resourcePartitions: {},
|
|
16
17
|
resourceWriteModes: {},
|
|
17
18
|
sinkWriteModes: {}
|
|
18
19
|
};
|
|
@@ -31,6 +32,7 @@ function selectSource(state, action) {
|
|
|
31
32
|
resourceReadModes: {},
|
|
32
33
|
resourceCursors: {},
|
|
33
34
|
resourceColumns: {},
|
|
35
|
+
resourcePartitions: {},
|
|
34
36
|
resourceWriteModes: {}
|
|
35
37
|
});
|
|
36
38
|
}
|
|
@@ -146,6 +148,19 @@ function setResourceColumns(state, action) {
|
|
|
146
148
|
}
|
|
147
149
|
};
|
|
148
150
|
}
|
|
151
|
+
function setResourcePartitions(state, action) {
|
|
152
|
+
const { sinkId, resource, partitions } = action.payload;
|
|
153
|
+
return {
|
|
154
|
+
...state,
|
|
155
|
+
resourcePartitions: {
|
|
156
|
+
...state.resourcePartitions,
|
|
157
|
+
[sinkId]: {
|
|
158
|
+
...state.resourcePartitions[sinkId],
|
|
159
|
+
[resource]: partitions
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
};
|
|
163
|
+
}
|
|
149
164
|
function setName(state, action) {
|
|
150
165
|
return {
|
|
151
166
|
...state,
|
|
@@ -232,6 +247,7 @@ var createPipelineModalReducer = (state, action) => {
|
|
|
232
247
|
case CreatePipelineModalActionType.SET_RESOURCE_READ_MODE: return setResourceReadMode(state, action);
|
|
233
248
|
case CreatePipelineModalActionType.SET_RESOURCE_CURSOR: return setResourceCursor(state, action);
|
|
234
249
|
case CreatePipelineModalActionType.SET_RESOURCE_COLUMNS: return setResourceColumns(state, action);
|
|
250
|
+
case CreatePipelineModalActionType.SET_RESOURCE_PARTITIONS: return setResourcePartitions(state, action);
|
|
235
251
|
case CreatePipelineModalActionType.SET_RESOURCE_WRITE_MODE: return setResourceWriteMode(state, action);
|
|
236
252
|
case CreatePipelineModalActionType.SET_SINK_WRITE_MODE: return setSinkWriteMode(state, action);
|
|
237
253
|
case CreatePipelineModalActionType.SET_NODE_CONFIG: return setNodeConfig(state, action);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ReadMode, WriteMode } from "../../../../gen/ingestion/v1/common_pb.js";
|
|
2
2
|
import { CREATE_PIPELINE_MODAL_DEFAULT_READ_MODE, CREATE_PIPELINE_MODAL_DEFAULT_WRITE_MODE } from "./constants.js";
|
|
3
|
-
import { getCompatibleWriteModes, getCursorOptions, getDefaultCursor, getDefaultReadMode, getEffectiveColumnSelection, getLockedColumnNames, getPipelineResourceStatus } from "../resource/utils.js";
|
|
3
|
+
import { getCompatibleWriteModes, getCursorOptions, getDefaultCursor, getDefaultReadMode, getEffectiveColumnSelection, getEffectivePartitionSelection, getLockedColumnNames, getPipelineResourceStatus } from "../resource/utils.js";
|
|
4
4
|
//#region src/pages/pipelines/components/create/rows.ts
|
|
5
5
|
var getDefaultPipelineName = (source, sinks) => {
|
|
6
6
|
const sinkNames = sinks.map((sink) => sink.name).join(", ");
|
|
@@ -11,6 +11,7 @@ var getDefaultPipelineName = (source, sinks) => {
|
|
|
11
11
|
var isResourceSelected = (selection, resource) => resource.isSelectable && (selection?.[resource.name] ?? resource.metadata.default_resources !== "false");
|
|
12
12
|
var createResourceRows = ({ state, sinkId, resources, columns, supportedReadModes, supportedWriteModes, hasReadLevers }) => {
|
|
13
13
|
const columnsByResource = new Map((columns?.resources ?? []).map((entry) => [entry.resource, entry.columns]));
|
|
14
|
+
const partitionsByResource = new Map((columns?.resources ?? []).map((entry) => [entry.resource, entry.partitions]));
|
|
14
15
|
const selectableResources = new Set((columns?.resources ?? []).filter((entry) => entry.selectableColumns).map((entry) => entry.resource));
|
|
15
16
|
const managedResources = new Set((columns?.resources ?? []).filter((entry) => entry.managedIncremental).map((entry) => entry.resource));
|
|
16
17
|
return resources.map((resource) => {
|
|
@@ -42,6 +43,11 @@ var createResourceRows = ({ state, sinkId, resources, columns, supportedReadMode
|
|
|
42
43
|
columns: resourceColumns,
|
|
43
44
|
locked: lockedColumns
|
|
44
45
|
});
|
|
46
|
+
const partitions = partitionsByResource.get(resource.name) ?? [];
|
|
47
|
+
const selectedPartitions = getEffectivePartitionSelection({
|
|
48
|
+
stored: state.resourcePartitions[sinkId]?.[resource.name],
|
|
49
|
+
partitions
|
|
50
|
+
});
|
|
45
51
|
return {
|
|
46
52
|
name: resource.name,
|
|
47
53
|
displayName: resource.displayName || resource.name,
|
|
@@ -58,6 +64,8 @@ var createResourceRows = ({ state, sinkId, resources, columns, supportedReadMode
|
|
|
58
64
|
selectedColumns,
|
|
59
65
|
lockedColumns,
|
|
60
66
|
isColumnSelectable: selectableResources.has(resource.name) || storedColumns !== void 0,
|
|
67
|
+
partitions,
|
|
68
|
+
selectedPartitions,
|
|
61
69
|
status: !hasReadLevers || !isSelected ? void 0 : getPipelineResourceStatus({
|
|
62
70
|
readMode,
|
|
63
71
|
readModeOptions,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ConnectorKind, ExecutionMode, ReadMode, ReplicationMode } from "../../../../gen/ingestion/v1/common_pb.js";
|
|
2
2
|
import { CreatePipelineRequestSchema, CreatePipelineVersionRequestSchema, PipelineEdgeSchema, PipelineNodeSchema, ResourceCursorConfigSchema } from "../../../../gen/ingestion/v1/pipelines_pb.js";
|
|
3
3
|
import { CreatePipelineNotifierRequestSchema } from "../../../../gen/ingestion/v1/notifiers_pb.js";
|
|
4
|
-
import { createResourceColumnSelections } from "../resource/utils.js";
|
|
4
|
+
import { createResourceColumnSelections, createResourcePartitionSelections } from "../resource/utils.js";
|
|
5
5
|
import { mapPipelineNotifierStateToInput } from "../notifier/utils.js";
|
|
6
6
|
import { parseWorkerConfiguration } from "../worker/utils.js";
|
|
7
7
|
import { create } from "@bufbuild/protobuf";
|
|
@@ -33,6 +33,11 @@ var createColumns = (row) => row.isColumnSelectable ? createResourceColumnSelect
|
|
|
33
33
|
selected: row.selectedColumns,
|
|
34
34
|
columns: row.columns
|
|
35
35
|
}) : [];
|
|
36
|
+
var createPartitions = (row, readMode) => readMode === ReadMode.INCREMENTAL ? [] : createResourcePartitionSelections({
|
|
37
|
+
resource: row.name,
|
|
38
|
+
selected: row.selectedPartitions,
|
|
39
|
+
partitions: row.partitions
|
|
40
|
+
});
|
|
36
41
|
var createSinkEdges = ({ sourceId, rows, sink, hasReadLevers }) => rows.filter((row) => row.isSelected).map((row) => {
|
|
37
42
|
const readMode = hasReadLevers ? row.readMode : ReadMode.UNSPECIFIED;
|
|
38
43
|
return create(PipelineEdgeSchema, {
|
|
@@ -42,7 +47,8 @@ var createSinkEdges = ({ sourceId, rows, sink, hasReadLevers }) => rows.filter((
|
|
|
42
47
|
readMode,
|
|
43
48
|
writeMode: hasReadLevers ? row.writeMode : sink.writeMode,
|
|
44
49
|
cursors: hasReadLevers ? createCursors([row], readMode) : [],
|
|
45
|
-
columns: createColumns(row)
|
|
50
|
+
columns: createColumns(row),
|
|
51
|
+
partitions: createPartitions(row, readMode)
|
|
46
52
|
});
|
|
47
53
|
});
|
|
48
54
|
var createEdges = ({ sourceConnection, rowsBySink, sinks, hasReadLevers }) => {
|