@deepnote/convert 2.1.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin.cjs +128 -56
- package/dist/bin.js +127 -57
- package/dist/index.cjs +25 -3
- package/dist/index.d.cts +334 -3
- package/dist/index.d.ts +334 -3
- package/dist/index.js +2 -2
- package/dist/{src-zxlAKpnq.js → src-C0XPNDnj.js} +673 -43
- package/dist/{src-1XGCb_OZ.cjs → src-CrnuwKOU.cjs} +803 -41
- package/package.json +3 -2
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
|
-
import { basename, dirname, extname, join } from "node:path";
|
|
3
|
-
import { createMarkdown, createPythonCode, deepnoteBlockSchema, deserializeDeepnoteFile, environmentSchema, executionSchema } from "@deepnote/blocks";
|
|
4
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { basename, dirname, extname, join, resolve } from "node:path";
|
|
3
|
+
import { createMarkdown, createPythonCode, deepnoteBlockSchema, deepnoteSnapshotSchema, deserializeDeepnoteFile, environmentSchema, executionSchema, isExecutableBlockType } from "@deepnote/blocks";
|
|
5
4
|
import { parse, stringify } from "yaml";
|
|
5
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
6
6
|
|
|
7
7
|
//#region src/utils.ts
|
|
8
8
|
/**
|
|
@@ -98,7 +98,7 @@ function sortKeysAlphabetically(obj) {
|
|
|
98
98
|
*/
|
|
99
99
|
function convertBlocksToJupyterNotebook(blocks, options) {
|
|
100
100
|
return {
|
|
101
|
-
cells: blocks.map((block) =>
|
|
101
|
+
cells: blocks.map((block) => convertBlockToJupyterCell(block)),
|
|
102
102
|
metadata: {
|
|
103
103
|
deepnote_notebook_id: options.notebookId,
|
|
104
104
|
deepnote_notebook_name: options.notebookName,
|
|
@@ -155,7 +155,27 @@ async function convertDeepnoteFileToJupyterFiles(deepnoteFilePath, options) {
|
|
|
155
155
|
await fs.writeFile(filePath, JSON.stringify(notebook, null, 2), "utf-8");
|
|
156
156
|
}
|
|
157
157
|
}
|
|
158
|
-
|
|
158
|
+
/**
|
|
159
|
+
* Converts a single Deepnote block to a Jupyter cell.
|
|
160
|
+
* This is useful for streaming export scenarios where blocks need to be
|
|
161
|
+
* processed one at a time rather than converting an entire notebook at once.
|
|
162
|
+
*
|
|
163
|
+
* @param block - A single DeepnoteBlock to convert
|
|
164
|
+
* @returns A JupyterCell object
|
|
165
|
+
*
|
|
166
|
+
* @example
|
|
167
|
+
* ```typescript
|
|
168
|
+
* import { convertBlockToJupyterCell } from '@deepnote/convert'
|
|
169
|
+
* import type { JupyterCell } from '@deepnote/convert'
|
|
170
|
+
*
|
|
171
|
+
* // Streaming export example
|
|
172
|
+
* for await (const block of blockStream) {
|
|
173
|
+
* const cell: JupyterCell = convertBlockToJupyterCell(block)
|
|
174
|
+
* outputStream.write(JSON.stringify(cell))
|
|
175
|
+
* }
|
|
176
|
+
* ```
|
|
177
|
+
*/
|
|
178
|
+
function convertBlockToJupyterCell(block) {
|
|
159
179
|
const content = block.content || "";
|
|
160
180
|
const jupyterCellType = convertBlockTypeToJupyter(block.type);
|
|
161
181
|
const executionStartedAt = "executionStartedAt" in block ? block.executionStartedAt : void 0;
|
|
@@ -623,11 +643,49 @@ function convertBlockToCell(block) {
|
|
|
623
643
|
//#region src/format-detection.ts
|
|
624
644
|
/** Check if file content is Marimo format */
|
|
625
645
|
function isMarimoContent(content) {
|
|
626
|
-
|
|
646
|
+
const lines = content.split("\n");
|
|
647
|
+
let inDocstring = false;
|
|
648
|
+
let docstringQuote = "";
|
|
649
|
+
const firstLine = lines.find((line) => {
|
|
650
|
+
const t = line.trim();
|
|
651
|
+
if (t.length === 0) return false;
|
|
652
|
+
if (t.startsWith("#!") || /^#\s*(-\*-\s*)?(coding|encoding)[:=]/i.test(t)) return false;
|
|
653
|
+
for (const q of ["\"\"\"", "'''"]) {
|
|
654
|
+
if (!inDocstring && t.startsWith(q)) {
|
|
655
|
+
if (t.length > 3 && t.endsWith(q)) return false;
|
|
656
|
+
inDocstring = true;
|
|
657
|
+
docstringQuote = q;
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
660
|
+
if (inDocstring && docstringQuote === q && t.includes(q)) {
|
|
661
|
+
inDocstring = false;
|
|
662
|
+
return false;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
if (inDocstring) return false;
|
|
666
|
+
return true;
|
|
667
|
+
}) ?? "";
|
|
668
|
+
return (/^import\s+marimo\b/.test(firstLine) || /^from\s+marimo\s+import\b/.test(firstLine)) && /@app\.cell\b/.test(content);
|
|
627
669
|
}
|
|
628
670
|
/** Check if file content is percent format */
|
|
629
671
|
function isPercentContent(content) {
|
|
630
|
-
|
|
672
|
+
const lines = content.split("\n");
|
|
673
|
+
let inTripleQuote = false;
|
|
674
|
+
let quoteChar = "";
|
|
675
|
+
for (const line of lines) {
|
|
676
|
+
for (const q of ["\"\"\"", "'''"]) {
|
|
677
|
+
let idx = line.indexOf(q, 0);
|
|
678
|
+
while (idx !== -1) {
|
|
679
|
+
if (!inTripleQuote) {
|
|
680
|
+
inTripleQuote = true;
|
|
681
|
+
quoteChar = q;
|
|
682
|
+
} else if (quoteChar === q) inTripleQuote = false;
|
|
683
|
+
idx = line.indexOf(q, idx + 3);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
if (!inTripleQuote && /^# %%/.test(line)) return true;
|
|
687
|
+
}
|
|
688
|
+
return false;
|
|
631
689
|
}
|
|
632
690
|
/**
|
|
633
691
|
* Detects the notebook format from filename and optionally content.
|
|
@@ -729,9 +787,14 @@ function convertJupyterNotebooksToDeepnote(notebooks, options) {
|
|
|
729
787
|
return deepnoteFile;
|
|
730
788
|
}
|
|
731
789
|
/**
|
|
732
|
-
*
|
|
790
|
+
* Reads and converts multiple Jupyter Notebook (.ipynb) files into a DeepnoteFile.
|
|
791
|
+
* This function reads the files and returns the converted DeepnoteFile without writing to disk.
|
|
792
|
+
*
|
|
793
|
+
* @param inputFilePaths - Array of paths to .ipynb files
|
|
794
|
+
* @param options - Conversion options including project name
|
|
795
|
+
* @returns A DeepnoteFile object
|
|
733
796
|
*/
|
|
734
|
-
async function
|
|
797
|
+
async function readAndConvertIpynbFiles(inputFilePaths, options) {
|
|
735
798
|
const notebooks = [];
|
|
736
799
|
for (const filePath of inputFilePaths) {
|
|
737
800
|
const notebook = await parseIpynbFile(filePath);
|
|
@@ -740,7 +803,13 @@ async function convertIpynbFilesToDeepnoteFile(inputFilePaths, options) {
|
|
|
740
803
|
notebook
|
|
741
804
|
});
|
|
742
805
|
}
|
|
743
|
-
|
|
806
|
+
return convertJupyterNotebooksToDeepnote(notebooks, { projectName: options.projectName });
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
809
|
+
* Converts multiple Jupyter Notebook (.ipynb) files into a single Deepnote project file.
|
|
810
|
+
*/
|
|
811
|
+
async function convertIpynbFilesToDeepnoteFile(inputFilePaths, options) {
|
|
812
|
+
const yamlContent = stringify(await readAndConvertIpynbFiles(inputFilePaths, { projectName: options.projectName }));
|
|
744
813
|
const parentDir = dirname(options.outputPath);
|
|
745
814
|
await fs.mkdir(parentDir, { recursive: true });
|
|
746
815
|
await fs.writeFile(options.outputPath, yamlContent, "utf-8");
|
|
@@ -784,7 +853,7 @@ function convertCellToBlock$3(cell, index, idGenerator) {
|
|
|
784
853
|
delete cell.block_group;
|
|
785
854
|
const executionCount = cell.execution_count ?? void 0;
|
|
786
855
|
const hasExecutionCount = executionCount !== void 0;
|
|
787
|
-
const hasOutputs = cell.cell_type === "code" && cell.outputs !== void 0;
|
|
856
|
+
const hasOutputs$1 = cell.cell_type === "code" && cell.outputs !== void 0;
|
|
788
857
|
return sortKeysAlphabetically(deepnoteBlockSchema.parse({
|
|
789
858
|
blockGroup,
|
|
790
859
|
content: source,
|
|
@@ -794,15 +863,444 @@ function convertCellToBlock$3(cell, index, idGenerator) {
|
|
|
794
863
|
...executionStartedAt ? { executionStartedAt } : {},
|
|
795
864
|
id: cellId ?? idGenerator(),
|
|
796
865
|
metadata: originalMetadata,
|
|
797
|
-
...hasOutputs ? { outputs: cell.outputs } : {},
|
|
866
|
+
...hasOutputs$1 ? { outputs: cell.outputs } : {},
|
|
798
867
|
sortingKey: sortingKey ?? createSortingKey(index),
|
|
799
868
|
type: blockType
|
|
800
869
|
}));
|
|
801
870
|
}
|
|
802
871
|
|
|
872
|
+
//#endregion
|
|
873
|
+
//#region src/snapshot/hash.ts
|
|
874
|
+
/**
|
|
875
|
+
* Computes a SHA-256 hash of the given content.
|
|
876
|
+
*
|
|
877
|
+
* @param content - The content to hash
|
|
878
|
+
* @returns Hash string in format 'sha256:{hex}'
|
|
879
|
+
*/
|
|
880
|
+
function computeContentHash(content) {
|
|
881
|
+
return `sha256:${createHash("sha256").update(content, "utf-8").digest("hex")}`;
|
|
882
|
+
}
|
|
883
|
+
/**
|
|
884
|
+
* Computes a snapshot hash from the file's key properties.
|
|
885
|
+
* The hash is based on: version, environment.hash, integrations, and all block contentHashes.
|
|
886
|
+
*
|
|
887
|
+
* @param file - The DeepnoteFile to compute hash for
|
|
888
|
+
* @returns Hash string in format 'sha256:{hex}'
|
|
889
|
+
*/
|
|
890
|
+
function computeSnapshotHash(file) {
|
|
891
|
+
const parts = [];
|
|
892
|
+
parts.push(`version:${file.version}`);
|
|
893
|
+
if (file.environment?.hash) parts.push(`env:${file.environment.hash}`);
|
|
894
|
+
const sortedIntegrations = [...file.project.integrations ?? []].sort((a, b) => a.id.localeCompare(b.id));
|
|
895
|
+
for (const integration of sortedIntegrations) parts.push(`integration:${integration.id}:${integration.type}`);
|
|
896
|
+
for (const notebook of file.project.notebooks) for (const block of notebook.blocks) if (block.contentHash) parts.push(`block:${block.id}:${block.contentHash}`);
|
|
897
|
+
const combined = parts.join("\n");
|
|
898
|
+
return `sha256:${createHash("sha256").update(combined, "utf-8").digest("hex")}`;
|
|
899
|
+
}
|
|
900
|
+
/**
|
|
901
|
+
* Adds content hashes to all blocks in a DeepnoteFile that don't already have them.
|
|
902
|
+
* Returns a new DeepnoteFile with hashes added (does not mutate the input).
|
|
903
|
+
*
|
|
904
|
+
* @param file - The DeepnoteFile to add hashes to
|
|
905
|
+
* @returns A new DeepnoteFile with content hashes added
|
|
906
|
+
*/
|
|
907
|
+
function addContentHashes(file) {
|
|
908
|
+
const result = structuredClone(file);
|
|
909
|
+
for (const notebook of result.project.notebooks) for (const block of notebook.blocks) if (!block.contentHash && block.content) block.contentHash = computeContentHash(block.content);
|
|
910
|
+
return result;
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
//#endregion
|
|
914
|
+
//#region src/snapshot/lookup.ts
|
|
915
|
+
/** Default directory name for snapshots */
|
|
916
|
+
const DEFAULT_SNAPSHOT_DIR = "snapshots";
|
|
917
|
+
/** Regex pattern for snapshot filenames */
|
|
918
|
+
const SNAPSHOT_FILENAME_PATTERN = /^(.+)_([0-9a-f-]{36})_(latest|[\dT:-]+)\.snapshot\.deepnote$/;
|
|
919
|
+
/**
|
|
920
|
+
* Parses a snapshot filename into its components.
|
|
921
|
+
*
|
|
922
|
+
* @param filename - The snapshot filename to parse
|
|
923
|
+
* @returns Parsed components or null if filename doesn't match pattern
|
|
924
|
+
*/
|
|
925
|
+
function parseSnapshotFilename(filename) {
|
|
926
|
+
const match = SNAPSHOT_FILENAME_PATTERN.exec(filename);
|
|
927
|
+
if (!match) return null;
|
|
928
|
+
return {
|
|
929
|
+
slug: match[1],
|
|
930
|
+
projectId: match[2],
|
|
931
|
+
timestamp: match[3]
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
/**
|
|
935
|
+
* Finds all snapshot files for a given project.
|
|
936
|
+
*
|
|
937
|
+
* @param projectDir - Directory containing the .deepnote file
|
|
938
|
+
* @param projectId - The project UUID to search for
|
|
939
|
+
* @param options - Snapshot options
|
|
940
|
+
* @returns Array of SnapshotInfo objects, sorted by timestamp (newest first)
|
|
941
|
+
*/
|
|
942
|
+
async function findSnapshotsForProject(projectDir, projectId, options = {}) {
|
|
943
|
+
const snapshotsPath = resolve(projectDir, options.snapshotDir ?? DEFAULT_SNAPSHOT_DIR);
|
|
944
|
+
try {
|
|
945
|
+
const entries = await fs.readdir(snapshotsPath, { withFileTypes: true });
|
|
946
|
+
const snapshots = [];
|
|
947
|
+
for (const entry of entries) {
|
|
948
|
+
if (!entry.isFile() || !entry.name.endsWith(".snapshot.deepnote")) continue;
|
|
949
|
+
const parsed = parseSnapshotFilename(entry.name);
|
|
950
|
+
if (parsed && parsed.projectId === projectId) snapshots.push({
|
|
951
|
+
path: join(snapshotsPath, entry.name),
|
|
952
|
+
slug: parsed.slug,
|
|
953
|
+
projectId: parsed.projectId,
|
|
954
|
+
timestamp: parsed.timestamp
|
|
955
|
+
});
|
|
956
|
+
}
|
|
957
|
+
snapshots.sort((a, b) => {
|
|
958
|
+
if (a.timestamp === "latest") return -1;
|
|
959
|
+
if (b.timestamp === "latest") return 1;
|
|
960
|
+
return b.timestamp.localeCompare(a.timestamp);
|
|
961
|
+
});
|
|
962
|
+
return snapshots;
|
|
963
|
+
} catch (err) {
|
|
964
|
+
if (err.code === "ENOENT") return [];
|
|
965
|
+
throw err;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
/**
|
|
969
|
+
* Loads the latest snapshot for a project.
|
|
970
|
+
*
|
|
971
|
+
* @param sourceFilePath - Path to the source .deepnote file
|
|
972
|
+
* @param projectId - The project UUID
|
|
973
|
+
* @param options - Snapshot options
|
|
974
|
+
* @returns The parsed DeepnoteSnapshot or null if not found
|
|
975
|
+
*/
|
|
976
|
+
async function loadLatestSnapshot(sourceFilePath, projectId, options = {}) {
|
|
977
|
+
const snapshots = await findSnapshotsForProject(dirname(sourceFilePath), projectId, options);
|
|
978
|
+
if (snapshots.length === 0) return null;
|
|
979
|
+
const snapshotInfo = snapshots[0];
|
|
980
|
+
return loadSnapshotFile(snapshotInfo.path);
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Loads and parses a snapshot file.
|
|
984
|
+
*
|
|
985
|
+
* @param snapshotPath - Path to the snapshot file
|
|
986
|
+
* @returns The parsed DeepnoteSnapshot
|
|
987
|
+
*/
|
|
988
|
+
async function loadSnapshotFile(snapshotPath) {
|
|
989
|
+
const parsed = parse(await fs.readFile(snapshotPath, "utf-8"));
|
|
990
|
+
return deepnoteSnapshotSchema.parse(parsed);
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* Gets the snapshot directory path for a source file.
|
|
994
|
+
*
|
|
995
|
+
* @param sourceFilePath - Path to the source .deepnote file
|
|
996
|
+
* @param options - Snapshot options
|
|
997
|
+
* @returns The snapshot directory path
|
|
998
|
+
*/
|
|
999
|
+
function getSnapshotDir(sourceFilePath, options = {}) {
|
|
1000
|
+
const snapshotDir = options.snapshotDir ?? DEFAULT_SNAPSHOT_DIR;
|
|
1001
|
+
return resolve(dirname(sourceFilePath), snapshotDir);
|
|
1002
|
+
}
|
|
1003
|
+
/**
|
|
1004
|
+
* Checks if a snapshot exists for a project.
|
|
1005
|
+
*
|
|
1006
|
+
* @param sourceFilePath - Path to the source .deepnote file
|
|
1007
|
+
* @param projectId - The project UUID
|
|
1008
|
+
* @param options - Snapshot options
|
|
1009
|
+
* @returns True if at least one snapshot exists
|
|
1010
|
+
*/
|
|
1011
|
+
async function snapshotExists(sourceFilePath, projectId, options = {}) {
|
|
1012
|
+
return (await findSnapshotsForProject(dirname(sourceFilePath), projectId, options)).length > 0;
|
|
1013
|
+
}
|
|
1014
|
+
/**
|
|
1015
|
+
* Extracts project information from a source file path.
|
|
1016
|
+
*
|
|
1017
|
+
* @param sourceFilePath - Path to the source .deepnote file
|
|
1018
|
+
* @returns Object with directory and filename without extension
|
|
1019
|
+
*/
|
|
1020
|
+
function parseSourceFilePath(sourceFilePath) {
|
|
1021
|
+
return {
|
|
1022
|
+
dir: dirname(sourceFilePath),
|
|
1023
|
+
name: basename(sourceFilePath).replace(/\.deepnote$/, "")
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
//#endregion
|
|
1028
|
+
//#region src/snapshot/marimo-outputs.ts
|
|
1029
|
+
/**
|
|
1030
|
+
* Finds the Marimo session cache file for a given .py file.
|
|
1031
|
+
* Looks in __marimo__/session/{filename}.json relative to the file's directory.
|
|
1032
|
+
*
|
|
1033
|
+
* @param marimoFilePath - Path to the .py Marimo file
|
|
1034
|
+
* @returns Path to the session cache file, or null if not found
|
|
1035
|
+
*/
|
|
1036
|
+
async function findMarimoSessionCache(marimoFilePath) {
|
|
1037
|
+
const sessionPath = join(dirname(marimoFilePath), "__marimo__", "session", `${basename(marimoFilePath)}.json`);
|
|
1038
|
+
try {
|
|
1039
|
+
await fs.access(sessionPath);
|
|
1040
|
+
return sessionPath;
|
|
1041
|
+
} catch {
|
|
1042
|
+
return null;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
/**
|
|
1046
|
+
* Reads and parses a Marimo session cache file.
|
|
1047
|
+
*
|
|
1048
|
+
* @param sessionPath - Path to the session cache JSON file
|
|
1049
|
+
* @returns The parsed session cache, or null if reading fails
|
|
1050
|
+
*/
|
|
1051
|
+
async function readMarimoSessionCache(sessionPath) {
|
|
1052
|
+
try {
|
|
1053
|
+
const content = await fs.readFile(sessionPath, "utf-8");
|
|
1054
|
+
return JSON.parse(content);
|
|
1055
|
+
} catch (_error) {
|
|
1056
|
+
return null;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
/**
|
|
1060
|
+
* Converts a Marimo session output to Jupyter/Deepnote output format.
|
|
1061
|
+
*
|
|
1062
|
+
* @param output - The Marimo session output
|
|
1063
|
+
* @returns A Jupyter-compatible output object
|
|
1064
|
+
*/
|
|
1065
|
+
function convertMarimoOutputToJupyter(output) {
|
|
1066
|
+
if (output.type === "error") return {
|
|
1067
|
+
output_type: "error",
|
|
1068
|
+
ename: output.ename || "Error",
|
|
1069
|
+
evalue: output.evalue || "",
|
|
1070
|
+
traceback: output.traceback || []
|
|
1071
|
+
};
|
|
1072
|
+
if (output.data) return {
|
|
1073
|
+
output_type: "execute_result",
|
|
1074
|
+
data: output.data,
|
|
1075
|
+
metadata: {},
|
|
1076
|
+
execution_count: null
|
|
1077
|
+
};
|
|
1078
|
+
return {
|
|
1079
|
+
output_type: "execute_result",
|
|
1080
|
+
data: { "text/plain": "" },
|
|
1081
|
+
metadata: {},
|
|
1082
|
+
execution_count: null
|
|
1083
|
+
};
|
|
1084
|
+
}
|
|
1085
|
+
/**
|
|
1086
|
+
* Converts Marimo console outputs to Jupyter stream outputs.
|
|
1087
|
+
*
|
|
1088
|
+
* @param consoleOutputs - Array of Marimo console outputs
|
|
1089
|
+
* @returns Array of Jupyter stream outputs
|
|
1090
|
+
*/
|
|
1091
|
+
function convertMarimoConsoleToJupyter(consoleOutputs) {
|
|
1092
|
+
const outputs = [];
|
|
1093
|
+
for (const consoleOutput of consoleOutputs) outputs.push({
|
|
1094
|
+
output_type: "stream",
|
|
1095
|
+
name: consoleOutput.channel,
|
|
1096
|
+
text: consoleOutput.data
|
|
1097
|
+
});
|
|
1098
|
+
return outputs;
|
|
1099
|
+
}
|
|
1100
|
+
/**
|
|
1101
|
+
* Converts a full Marimo session cell to Jupyter outputs.
|
|
1102
|
+
*
|
|
1103
|
+
* @param cell - The Marimo session cell
|
|
1104
|
+
* @returns Array of Jupyter-compatible outputs
|
|
1105
|
+
*/
|
|
1106
|
+
function convertMarimoSessionCellToOutputs(cell) {
|
|
1107
|
+
const outputs = [];
|
|
1108
|
+
outputs.push(...convertMarimoConsoleToJupyter(cell.console ?? []));
|
|
1109
|
+
for (const output of cell.outputs) outputs.push(convertMarimoOutputToJupyter(output));
|
|
1110
|
+
return outputs;
|
|
1111
|
+
}
|
|
1112
|
+
/**
|
|
1113
|
+
* Gets outputs from the Marimo session cache file.
|
|
1114
|
+
* This is the preferred method as it doesn't require the marimo CLI.
|
|
1115
|
+
* Outputs are keyed by code_hash for reliable matching even when cells are reordered.
|
|
1116
|
+
*
|
|
1117
|
+
* @param marimoFilePath - Path to the .py Marimo file
|
|
1118
|
+
* @returns Map of code_hash to outputs, or null if session cache is not found
|
|
1119
|
+
*/
|
|
1120
|
+
async function getMarimoOutputsFromCache(marimoFilePath) {
|
|
1121
|
+
const sessionPath = await findMarimoSessionCache(marimoFilePath);
|
|
1122
|
+
if (!sessionPath) return null;
|
|
1123
|
+
const cache = await readMarimoSessionCache(sessionPath);
|
|
1124
|
+
if (!cache) return null;
|
|
1125
|
+
const outputMap = /* @__PURE__ */ new Map();
|
|
1126
|
+
for (const cell of cache.cells) {
|
|
1127
|
+
const outputs = convertMarimoSessionCellToOutputs(cell);
|
|
1128
|
+
if (outputs.length > 0) {
|
|
1129
|
+
const nonEmptyOutputs = outputs.filter((o) => {
|
|
1130
|
+
if (o.output_type === "execute_result" && o.data) {
|
|
1131
|
+
const data = o.data;
|
|
1132
|
+
return Object.values(data).some((v) => {
|
|
1133
|
+
if (typeof v === "string") return v.trim() !== "";
|
|
1134
|
+
return v != null;
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
return true;
|
|
1138
|
+
});
|
|
1139
|
+
if (nonEmptyOutputs.length > 0) outputMap.set(cell.code_hash, nonEmptyOutputs);
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
return outputMap;
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
//#endregion
|
|
1146
|
+
//#region src/snapshot/merge.ts
|
|
1147
|
+
/**
|
|
1148
|
+
* Merges outputs from a snapshot into a source file.
|
|
1149
|
+
* Returns a new DeepnoteFile with outputs added from the snapshot.
|
|
1150
|
+
*
|
|
1151
|
+
* @param source - The source DeepnoteFile (without outputs)
|
|
1152
|
+
* @param snapshot - The snapshot containing outputs
|
|
1153
|
+
* @param options - Merge options
|
|
1154
|
+
* @returns A new DeepnoteFile with outputs merged in
|
|
1155
|
+
*/
|
|
1156
|
+
function mergeSnapshotIntoSource(source, snapshot, options = {}) {
|
|
1157
|
+
const { skipMismatched = false } = options;
|
|
1158
|
+
const outputMap = /* @__PURE__ */ new Map();
|
|
1159
|
+
for (const notebook of snapshot.project.notebooks) for (const block of notebook.blocks) {
|
|
1160
|
+
const execBlock = block;
|
|
1161
|
+
if (execBlock.outputs && execBlock.outputs.length > 0) outputMap.set(block.id, {
|
|
1162
|
+
contentHash: block.contentHash,
|
|
1163
|
+
executionCount: execBlock.executionCount,
|
|
1164
|
+
executionStartedAt: execBlock.executionStartedAt,
|
|
1165
|
+
executionFinishedAt: execBlock.executionFinishedAt,
|
|
1166
|
+
outputs: execBlock.outputs
|
|
1167
|
+
});
|
|
1168
|
+
}
|
|
1169
|
+
return {
|
|
1170
|
+
...source,
|
|
1171
|
+
environment: snapshot.environment ?? source.environment,
|
|
1172
|
+
execution: snapshot.execution ?? source.execution,
|
|
1173
|
+
project: {
|
|
1174
|
+
...source.project,
|
|
1175
|
+
notebooks: source.project.notebooks.map((notebook) => ({
|
|
1176
|
+
...notebook,
|
|
1177
|
+
blocks: notebook.blocks.map((block) => {
|
|
1178
|
+
const snapshotData = outputMap.get(block.id);
|
|
1179
|
+
if (!snapshotData) return block;
|
|
1180
|
+
if (skipMismatched && snapshotData.contentHash && block.contentHash) {
|
|
1181
|
+
if (snapshotData.contentHash !== block.contentHash) return block;
|
|
1182
|
+
}
|
|
1183
|
+
return {
|
|
1184
|
+
...block,
|
|
1185
|
+
...snapshotData.executionCount !== void 0 ? { executionCount: snapshotData.executionCount } : {},
|
|
1186
|
+
...snapshotData.executionStartedAt ? { executionStartedAt: snapshotData.executionStartedAt } : {},
|
|
1187
|
+
...snapshotData.executionFinishedAt ? { executionFinishedAt: snapshotData.executionFinishedAt } : {},
|
|
1188
|
+
...snapshotData.outputs ? { outputs: snapshotData.outputs } : {}
|
|
1189
|
+
};
|
|
1190
|
+
})
|
|
1191
|
+
}))
|
|
1192
|
+
}
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
/**
|
|
1196
|
+
* Counts blocks with outputs in a file.
|
|
1197
|
+
*
|
|
1198
|
+
* @param file - The DeepnoteFile to count outputs in
|
|
1199
|
+
* @returns Number of blocks that have outputs
|
|
1200
|
+
*/
|
|
1201
|
+
function countBlocksWithOutputs(file) {
|
|
1202
|
+
let count = 0;
|
|
1203
|
+
for (const notebook of file.project.notebooks) for (const block of notebook.blocks) {
|
|
1204
|
+
const execBlock = block;
|
|
1205
|
+
if (execBlock.outputs && execBlock.outputs.length > 0) count++;
|
|
1206
|
+
}
|
|
1207
|
+
return count;
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
//#endregion
|
|
1211
|
+
//#region src/snapshot/split.ts
|
|
1212
|
+
/**
|
|
1213
|
+
* Creates a slug from a project name.
|
|
1214
|
+
* Normalizes accented characters to ASCII equivalents (e.g., é → e),
|
|
1215
|
+
* converts to lowercase, replaces spaces and special chars with hyphens,
|
|
1216
|
+
* removes consecutive hyphens, and trims leading/trailing hyphens.
|
|
1217
|
+
*
|
|
1218
|
+
* @param name - The project name to slugify
|
|
1219
|
+
* @returns A URL-safe slug
|
|
1220
|
+
*/
|
|
1221
|
+
function slugifyProjectName(name) {
|
|
1222
|
+
return name.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
1223
|
+
}
|
|
1224
|
+
/**
|
|
1225
|
+
* Generates a snapshot filename from project info.
|
|
1226
|
+
*
|
|
1227
|
+
* @param slug - The project name slug
|
|
1228
|
+
* @param projectId - The project UUID
|
|
1229
|
+
* @param timestamp - Timestamp string or 'latest'
|
|
1230
|
+
* @returns Filename in format '{slug}_{projectId}_{timestamp}.snapshot.deepnote'
|
|
1231
|
+
*/
|
|
1232
|
+
function generateSnapshotFilename(slug, projectId, timestamp = "latest") {
|
|
1233
|
+
return `${slug}_${projectId}_${timestamp}.snapshot.deepnote`;
|
|
1234
|
+
}
|
|
1235
|
+
/**
|
|
1236
|
+
* Removes output-related fields from a block, returning a clean source block.
|
|
1237
|
+
*/
|
|
1238
|
+
function stripOutputsFromBlock(block) {
|
|
1239
|
+
if (!isExecutableBlockType(block.type)) return block;
|
|
1240
|
+
const { executionCount: _executionCount, executionStartedAt: _executionStartedAt, executionFinishedAt: _executionFinishedAt, outputs: _outputs,...rest } = block;
|
|
1241
|
+
return rest;
|
|
1242
|
+
}
|
|
1243
|
+
/**
|
|
1244
|
+
* Splits a DeepnoteFile into a source file (no outputs) and a snapshot file (outputs only).
|
|
1245
|
+
*
|
|
1246
|
+
* @param file - The complete DeepnoteFile with outputs
|
|
1247
|
+
* @returns Object containing source and snapshot files
|
|
1248
|
+
*/
|
|
1249
|
+
function splitDeepnoteFile(file) {
|
|
1250
|
+
const fileWithHashes = addContentHashes(file);
|
|
1251
|
+
const snapshotHash = computeSnapshotHash(fileWithHashes);
|
|
1252
|
+
const { snapshotHash: _snapshotHash,...sourceMetadata } = fileWithHashes.metadata ?? {};
|
|
1253
|
+
return {
|
|
1254
|
+
source: {
|
|
1255
|
+
...fileWithHashes,
|
|
1256
|
+
metadata: sourceMetadata,
|
|
1257
|
+
project: {
|
|
1258
|
+
...fileWithHashes.project,
|
|
1259
|
+
notebooks: fileWithHashes.project.notebooks.map((notebook) => ({
|
|
1260
|
+
...notebook,
|
|
1261
|
+
blocks: notebook.blocks.map(stripOutputsFromBlock)
|
|
1262
|
+
}))
|
|
1263
|
+
}
|
|
1264
|
+
},
|
|
1265
|
+
snapshot: {
|
|
1266
|
+
...fileWithHashes,
|
|
1267
|
+
environment: fileWithHashes.environment ?? {},
|
|
1268
|
+
execution: fileWithHashes.execution ?? {},
|
|
1269
|
+
metadata: {
|
|
1270
|
+
...fileWithHashes.metadata,
|
|
1271
|
+
snapshotHash
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
};
|
|
1275
|
+
}
|
|
1276
|
+
/**
|
|
1277
|
+
* Checks if a DeepnoteFile has any outputs.
|
|
1278
|
+
*
|
|
1279
|
+
* @param file - The DeepnoteFile to check
|
|
1280
|
+
* @returns True if any block has outputs
|
|
1281
|
+
*/
|
|
1282
|
+
function hasOutputs(file) {
|
|
1283
|
+
for (const notebook of file.project.notebooks) for (const block of notebook.blocks) {
|
|
1284
|
+
if (!isExecutableBlockType(block.type)) continue;
|
|
1285
|
+
const execBlock = block;
|
|
1286
|
+
if (execBlock.outputs && execBlock.outputs.length > 0) return true;
|
|
1287
|
+
}
|
|
1288
|
+
return false;
|
|
1289
|
+
}
|
|
1290
|
+
|
|
803
1291
|
//#endregion
|
|
804
1292
|
//#region src/marimo-to-deepnote.ts
|
|
805
1293
|
/**
|
|
1294
|
+
* Computes a code hash for a cell's content.
|
|
1295
|
+
* This matches how Marimo computes code_hash for the session cache.
|
|
1296
|
+
*
|
|
1297
|
+
* @param content - The cell's code content
|
|
1298
|
+
* @returns An MD5 hash string (32 hex chars)
|
|
1299
|
+
*/
|
|
1300
|
+
function computeCodeHash(content) {
|
|
1301
|
+
return createHash("md5").update(content, "utf-8").digest("hex");
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
806
1304
|
* Splits a string on commas that are at the top level (not inside parentheses, brackets, braces, or string literals).
|
|
807
1305
|
* This handles cases like "func(a, b), other" and 'return "a,b", x' correctly.
|
|
808
1306
|
* Supports single quotes, double quotes, and backticks, with proper escape handling.
|
|
@@ -873,13 +1371,26 @@ function parseMarimoFormat(content) {
|
|
|
873
1371
|
const generatedWith = /__generated_with\s*=\s*["']([^"']+)["']/.exec(content)?.[1];
|
|
874
1372
|
const width = /(?:marimo|mo)\.App\([^)]*width\s*=\s*["']([^"']+)["']/.exec(content)?.[1];
|
|
875
1373
|
const title = /(?:marimo|mo)\.App\([^)]*title\s*=\s*["']([^"']+)["']/.exec(content)?.[1];
|
|
876
|
-
const cellRegex = /@app\.cell(?:\(([^)]*)\))?\s*\n\s*def\s+(\w+)\s*\(([^)]*)\)\s*(?:->.*?)?\s*:\s*\n([\s\S]*?)(?=@app\.cell|if\s+__name__|$)/g;
|
|
1374
|
+
const cellRegex = /@app\.(cell|function)(?:\(([^)]*)\))?\s*\n\s*def\s+(\w+)\s*\(([^)]*)\)\s*(?:->.*?)?\s*:\s*\n([\s\S]*?)(?=@app\.cell|@app\.function|if\s+__name__|$)/g;
|
|
877
1375
|
let match = cellRegex.exec(content);
|
|
878
1376
|
while (match !== null) {
|
|
879
|
-
const
|
|
880
|
-
const
|
|
881
|
-
const
|
|
882
|
-
|
|
1377
|
+
const decoratorType = match[1];
|
|
1378
|
+
const decoratorArgs = match[2] || "";
|
|
1379
|
+
const functionName = match[3];
|
|
1380
|
+
const params = match[4].trim();
|
|
1381
|
+
let body = match[5];
|
|
1382
|
+
if (decoratorType === "function") {
|
|
1383
|
+
const funcDef = `def ${functionName}(${params})${/@app\.function(?:\([^)]*\))?\s*\n\s*def\s+\w+\s*\([^)]*\)\s*(->.*?)?\s*:/.exec(content.slice(match.index))?.[1] || ""}:\n${body}`;
|
|
1384
|
+
cells.push({
|
|
1385
|
+
cellType: "code",
|
|
1386
|
+
content: funcDef.trim(),
|
|
1387
|
+
functionName,
|
|
1388
|
+
hidden: /hide_code\s*=\s*True/.test(decoratorArgs),
|
|
1389
|
+
disabled: /disabled\s*=\s*True/.test(decoratorArgs)
|
|
1390
|
+
});
|
|
1391
|
+
match = cellRegex.exec(content);
|
|
1392
|
+
continue;
|
|
1393
|
+
}
|
|
883
1394
|
const dependencies = params ? params.split(",").map((p) => p.trim()).filter((p) => p.length > 0) : void 0;
|
|
884
1395
|
const hidden = /hide_code\s*=\s*True/.test(decoratorArgs);
|
|
885
1396
|
const disabled = /disabled\s*=\s*True/.test(decoratorArgs);
|
|
@@ -967,41 +1478,70 @@ function parseMarimoFormat(content) {
|
|
|
967
1478
|
* This is the lowest-level conversion function.
|
|
968
1479
|
*
|
|
969
1480
|
* @param app - The Marimo app object to convert
|
|
970
|
-
* @param options - Optional conversion options including custom ID generator
|
|
1481
|
+
* @param options - Optional conversion options including custom ID generator and outputs
|
|
971
1482
|
* @returns Array of DeepnoteBlock objects
|
|
972
1483
|
*/
|
|
973
1484
|
function convertMarimoAppToBlocks(app, options) {
|
|
974
1485
|
const idGenerator = options?.idGenerator ?? randomUUID;
|
|
975
|
-
|
|
1486
|
+
const outputs = options?.outputs;
|
|
1487
|
+
return app.cells.map((cell, index) => {
|
|
1488
|
+
const codeHash = computeCodeHash(cell.content);
|
|
1489
|
+
const cellOutputs = outputs?.get(codeHash);
|
|
1490
|
+
return convertCellToBlock$2(cell, index, idGenerator, cellOutputs);
|
|
1491
|
+
});
|
|
976
1492
|
}
|
|
977
1493
|
/**
|
|
978
|
-
*
|
|
979
|
-
* This is a
|
|
980
|
-
*
|
|
981
|
-
* @param apps - Array of Marimo apps with filenames
|
|
982
|
-
* @param options - Conversion options including project name and optional ID generator
|
|
983
|
-
* @returns A DeepnoteFile object
|
|
1494
|
+
* Creates a base DeepnoteFile structure with empty notebooks.
|
|
1495
|
+
* This is a helper to reduce duplication in conversion functions.
|
|
984
1496
|
*/
|
|
985
|
-
function
|
|
986
|
-
|
|
987
|
-
const firstNotebookId = apps.length > 0 ? idGenerator() : void 0;
|
|
988
|
-
const deepnoteFile = {
|
|
1497
|
+
function createDeepnoteFileSkeleton(projectName, idGenerator, firstNotebookId) {
|
|
1498
|
+
return {
|
|
989
1499
|
metadata: { createdAt: (/* @__PURE__ */ new Date()).toISOString() },
|
|
990
1500
|
project: {
|
|
991
1501
|
id: idGenerator(),
|
|
992
1502
|
initNotebookId: firstNotebookId,
|
|
993
1503
|
integrations: [],
|
|
994
|
-
name:
|
|
1504
|
+
name: projectName,
|
|
995
1505
|
notebooks: [],
|
|
996
1506
|
settings: {}
|
|
997
1507
|
},
|
|
998
1508
|
version: "1.0.0"
|
|
999
1509
|
};
|
|
1510
|
+
}
|
|
1511
|
+
/**
|
|
1512
|
+
* Converts Marimo app objects into a Deepnote project file.
|
|
1513
|
+
* This is a pure conversion function that doesn't perform any file I/O.
|
|
1514
|
+
*
|
|
1515
|
+
* @param apps - Array of Marimo apps with filenames
|
|
1516
|
+
* @param options - Conversion options including project name and optional ID generator
|
|
1517
|
+
* @returns A DeepnoteFile object
|
|
1518
|
+
*/
|
|
1519
|
+
function convertMarimoAppsToDeepnote(apps, options) {
|
|
1520
|
+
return convertMarimoAppsToDeepnoteFile(apps.map((app) => ({
|
|
1521
|
+
...app,
|
|
1522
|
+
outputs: void 0
|
|
1523
|
+
})), options);
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* Converts Marimo app objects with outputs into a Deepnote project file.
|
|
1527
|
+
* This variant includes outputs from the Marimo session cache.
|
|
1528
|
+
*
|
|
1529
|
+
* @param apps - Array of Marimo apps with filenames and optional outputs
|
|
1530
|
+
* @param options - Conversion options including project name and optional ID generator
|
|
1531
|
+
* @returns A DeepnoteFile object
|
|
1532
|
+
*/
|
|
1533
|
+
function convertMarimoAppsToDeepnoteFile(apps, options) {
|
|
1534
|
+
const idGenerator = options.idGenerator ?? randomUUID;
|
|
1535
|
+
const firstNotebookId = apps.length > 0 ? idGenerator() : void 0;
|
|
1536
|
+
const deepnoteFile = createDeepnoteFileSkeleton(options.projectName, idGenerator, firstNotebookId);
|
|
1000
1537
|
for (let i = 0; i < apps.length; i++) {
|
|
1001
|
-
const { filename, app } = apps[i];
|
|
1538
|
+
const { filename, app, outputs } = apps[i];
|
|
1002
1539
|
const filenameWithoutExt = basename(filename, extname(filename)) || "Untitled notebook";
|
|
1003
1540
|
const notebookName = app.title || filenameWithoutExt;
|
|
1004
|
-
const blocks = convertMarimoAppToBlocks(app, {
|
|
1541
|
+
const blocks = convertMarimoAppToBlocks(app, {
|
|
1542
|
+
idGenerator,
|
|
1543
|
+
outputs
|
|
1544
|
+
});
|
|
1005
1545
|
const notebookId = i === 0 && firstNotebookId ? firstNotebookId : idGenerator();
|
|
1006
1546
|
deepnoteFile.project.notebooks.push({
|
|
1007
1547
|
blocks,
|
|
@@ -1014,15 +1554,45 @@ function convertMarimoAppsToDeepnote(apps, options) {
|
|
|
1014
1554
|
return deepnoteFile;
|
|
1015
1555
|
}
|
|
1016
1556
|
/**
|
|
1557
|
+
* Reads and converts multiple Marimo (.py) files into a DeepnoteFile.
|
|
1558
|
+
* This function reads the files and returns the converted DeepnoteFile without writing to disk.
|
|
1559
|
+
*
|
|
1560
|
+
* @param inputFilePaths - Array of paths to Marimo .py files
|
|
1561
|
+
* @param options - Conversion options including project name
|
|
1562
|
+
* @returns A DeepnoteFile object
|
|
1563
|
+
*/
|
|
1564
|
+
async function readAndConvertMarimoFiles(inputFilePaths, options) {
|
|
1565
|
+
const apps = [];
|
|
1566
|
+
for (const filePath of inputFilePaths) try {
|
|
1567
|
+
const app = parseMarimoFormat(await fs.readFile(filePath, "utf-8"));
|
|
1568
|
+
const outputs = await getMarimoOutputsFromCache(filePath);
|
|
1569
|
+
apps.push({
|
|
1570
|
+
filename: basename(filePath),
|
|
1571
|
+
app,
|
|
1572
|
+
outputs: outputs ?? void 0
|
|
1573
|
+
});
|
|
1574
|
+
} catch (err) {
|
|
1575
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
1576
|
+
const errorStack = err instanceof Error ? err.stack : void 0;
|
|
1577
|
+
throw new Error(`Failed to read or parse file ${basename(filePath)}: ${errorMessage}`, { cause: errorStack ? {
|
|
1578
|
+
originalError: err,
|
|
1579
|
+
stack: errorStack
|
|
1580
|
+
} : err });
|
|
1581
|
+
}
|
|
1582
|
+
return convertMarimoAppsToDeepnoteFile(apps, { projectName: options.projectName });
|
|
1583
|
+
}
|
|
1584
|
+
/**
|
|
1017
1585
|
* Converts multiple Marimo (.py) files into a single Deepnote project file.
|
|
1018
1586
|
*/
|
|
1019
1587
|
async function convertMarimoFilesToDeepnoteFile(inputFilePaths, options) {
|
|
1020
1588
|
const apps = [];
|
|
1021
1589
|
for (const filePath of inputFilePaths) try {
|
|
1022
1590
|
const app = parseMarimoFormat(await fs.readFile(filePath, "utf-8"));
|
|
1591
|
+
const outputs = await getMarimoOutputsFromCache(filePath);
|
|
1023
1592
|
apps.push({
|
|
1024
1593
|
filename: basename(filePath),
|
|
1025
|
-
app
|
|
1594
|
+
app,
|
|
1595
|
+
outputs: outputs ?? void 0
|
|
1026
1596
|
});
|
|
1027
1597
|
} catch (err) {
|
|
1028
1598
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
@@ -1032,12 +1602,12 @@ async function convertMarimoFilesToDeepnoteFile(inputFilePaths, options) {
|
|
|
1032
1602
|
stack: errorStack
|
|
1033
1603
|
} : err });
|
|
1034
1604
|
}
|
|
1035
|
-
const yamlContent = stringify(
|
|
1605
|
+
const yamlContent = stringify(convertMarimoAppsToDeepnoteFile(apps, { projectName: options.projectName }));
|
|
1036
1606
|
const parentDir = dirname(options.outputPath);
|
|
1037
1607
|
await fs.mkdir(parentDir, { recursive: true });
|
|
1038
1608
|
await fs.writeFile(options.outputPath, yamlContent, "utf-8");
|
|
1039
1609
|
}
|
|
1040
|
-
function convertCellToBlock$2(cell, index, idGenerator) {
|
|
1610
|
+
function convertCellToBlock$2(cell, index, idGenerator, outputs) {
|
|
1041
1611
|
let blockType;
|
|
1042
1612
|
if (cell.cellType === "markdown") blockType = "markdown";
|
|
1043
1613
|
else if (cell.cellType === "sql") blockType = "sql";
|
|
@@ -1055,7 +1625,8 @@ function convertCellToBlock$2(cell, index, idGenerator) {
|
|
|
1055
1625
|
id: idGenerator(),
|
|
1056
1626
|
metadata: Object.keys(metadata).length > 0 ? metadata : {},
|
|
1057
1627
|
sortingKey: createSortingKey(index),
|
|
1058
|
-
type: blockType
|
|
1628
|
+
type: blockType,
|
|
1629
|
+
...outputs && outputs.length > 0 && (blockType === "code" || blockType === "sql") ? { outputs } : {}
|
|
1059
1630
|
};
|
|
1060
1631
|
}
|
|
1061
1632
|
|
|
@@ -1182,9 +1753,14 @@ function convertPercentNotebooksToDeepnote(notebooks, options) {
|
|
|
1182
1753
|
return deepnoteFile;
|
|
1183
1754
|
}
|
|
1184
1755
|
/**
|
|
1185
|
-
*
|
|
1756
|
+
* Reads and converts multiple percent format (.py) files into a DeepnoteFile.
|
|
1757
|
+
* This function reads the files and returns the converted DeepnoteFile without writing to disk.
|
|
1758
|
+
*
|
|
1759
|
+
* @param inputFilePaths - Array of paths to percent format .py files
|
|
1760
|
+
* @param options - Conversion options including project name
|
|
1761
|
+
* @returns A DeepnoteFile object
|
|
1186
1762
|
*/
|
|
1187
|
-
async function
|
|
1763
|
+
async function readAndConvertPercentFiles(inputFilePaths, options) {
|
|
1188
1764
|
const notebooks = [];
|
|
1189
1765
|
for (const filePath of inputFilePaths) {
|
|
1190
1766
|
const notebook = parsePercentFormat(await fs.readFile(filePath, "utf-8"));
|
|
@@ -1193,7 +1769,13 @@ async function convertPercentFilesToDeepnoteFile(inputFilePaths, options) {
|
|
|
1193
1769
|
notebook
|
|
1194
1770
|
});
|
|
1195
1771
|
}
|
|
1196
|
-
|
|
1772
|
+
return convertPercentNotebooksToDeepnote(notebooks, { projectName: options.projectName });
|
|
1773
|
+
}
|
|
1774
|
+
/**
|
|
1775
|
+
* Converts multiple percent format (.py) files into a single Deepnote project file.
|
|
1776
|
+
*/
|
|
1777
|
+
async function convertPercentFilesToDeepnoteFile(inputFilePaths, options) {
|
|
1778
|
+
const yamlContent = stringify(await readAndConvertPercentFiles(inputFilePaths, { projectName: options.projectName }));
|
|
1197
1779
|
const parentDir = dirname(options.outputPath);
|
|
1198
1780
|
await fs.mkdir(parentDir, { recursive: true });
|
|
1199
1781
|
await fs.writeFile(options.outputPath, yamlContent, "utf-8");
|
|
@@ -1444,9 +2026,14 @@ function convertQuartoDocumentsToDeepnote(documents, options) {
|
|
|
1444
2026
|
return deepnoteFile;
|
|
1445
2027
|
}
|
|
1446
2028
|
/**
|
|
1447
|
-
*
|
|
2029
|
+
* Reads and converts multiple Quarto (.qmd) files into a DeepnoteFile.
|
|
2030
|
+
* This function reads the files and returns the converted DeepnoteFile without writing to disk.
|
|
2031
|
+
*
|
|
2032
|
+
* @param inputFilePaths - Array of paths to .qmd files
|
|
2033
|
+
* @param options - Conversion options including project name
|
|
2034
|
+
* @returns A DeepnoteFile object
|
|
1448
2035
|
*/
|
|
1449
|
-
async function
|
|
2036
|
+
async function readAndConvertQuartoFiles(inputFilePaths, options) {
|
|
1450
2037
|
const documents = [];
|
|
1451
2038
|
for (const filePath of inputFilePaths) {
|
|
1452
2039
|
const document = parseQuartoFormat(await fs.readFile(filePath, "utf-8"));
|
|
@@ -1455,7 +2042,13 @@ async function convertQuartoFilesToDeepnoteFile(inputFilePaths, options) {
|
|
|
1455
2042
|
document
|
|
1456
2043
|
});
|
|
1457
2044
|
}
|
|
1458
|
-
|
|
2045
|
+
return convertQuartoDocumentsToDeepnote(documents, { projectName: options.projectName });
|
|
2046
|
+
}
|
|
2047
|
+
/**
|
|
2048
|
+
* Converts multiple Quarto (.qmd) files into a single Deepnote project file.
|
|
2049
|
+
*/
|
|
2050
|
+
async function convertQuartoFilesToDeepnoteFile(inputFilePaths, options) {
|
|
2051
|
+
const yamlContent = stringify(await readAndConvertQuartoFiles(inputFilePaths, { projectName: options.projectName }));
|
|
1459
2052
|
const parentDir = dirname(options.outputPath);
|
|
1460
2053
|
await fs.mkdir(parentDir, { recursive: true });
|
|
1461
2054
|
await fs.writeFile(options.outputPath, yamlContent, "utf-8");
|
|
@@ -1482,4 +2075,41 @@ function convertCellToBlock(cell, index, idGenerator) {
|
|
|
1482
2075
|
}
|
|
1483
2076
|
|
|
1484
2077
|
//#endregion
|
|
1485
|
-
|
|
2078
|
+
//#region src/write-deepnote-file.ts
|
|
2079
|
+
/**
|
|
2080
|
+
* Writes a DeepnoteFile to disk, optionally splitting outputs into a snapshot file.
|
|
2081
|
+
*
|
|
2082
|
+
* When singleFile is false (default) and the file contains outputs:
|
|
2083
|
+
* - Splits the file in memory into source (no outputs) and snapshot (with outputs)
|
|
2084
|
+
* - Writes both files in parallel
|
|
2085
|
+
*
|
|
2086
|
+
* When singleFile is true or there are no outputs:
|
|
2087
|
+
* - Writes the complete file as-is
|
|
2088
|
+
*
|
|
2089
|
+
* @param options - Write options including the file, output path, and project name
|
|
2090
|
+
* @returns Object containing paths to the written files
|
|
2091
|
+
*/
|
|
2092
|
+
async function writeDeepnoteFile(options) {
|
|
2093
|
+
const { file, outputPath, projectName, singleFile = false } = options;
|
|
2094
|
+
const parentDir = dirname(outputPath);
|
|
2095
|
+
await fs.mkdir(parentDir, { recursive: true });
|
|
2096
|
+
if (singleFile || !hasOutputs(file)) {
|
|
2097
|
+
const yamlContent = stringify(file);
|
|
2098
|
+
await fs.writeFile(outputPath, yamlContent, "utf-8");
|
|
2099
|
+
return { sourcePath: outputPath };
|
|
2100
|
+
}
|
|
2101
|
+
const { source, snapshot } = splitDeepnoteFile(file);
|
|
2102
|
+
const snapshotDir = getSnapshotDir(outputPath);
|
|
2103
|
+
const snapshotPath = resolve(snapshotDir, generateSnapshotFilename(slugifyProjectName(projectName) || "project", file.project.id));
|
|
2104
|
+
const sourceYaml = stringify(source);
|
|
2105
|
+
const snapshotYaml = stringify(snapshot);
|
|
2106
|
+
await fs.mkdir(snapshotDir, { recursive: true });
|
|
2107
|
+
await Promise.all([fs.writeFile(outputPath, sourceYaml, "utf-8"), fs.writeFile(snapshotPath, snapshotYaml, "utf-8")]);
|
|
2108
|
+
return {
|
|
2109
|
+
sourcePath: outputPath,
|
|
2110
|
+
snapshotPath
|
|
2111
|
+
};
|
|
2112
|
+
}
|
|
2113
|
+
|
|
2114
|
+
//#endregion
|
|
2115
|
+
export { convertBlocksToJupyterNotebook as $, addContentHashes as A, convertBlocksToQuartoDocument as B, findSnapshotsForProject as C, parseSnapshotFilename as D, loadSnapshotFile as E, convertJupyterNotebooksToDeepnote as F, convertDeepnoteFileToPercentFiles as G, convertDeepnoteToQuartoDocuments as H, readAndConvertIpynbFiles as I, convertBlocksToMarimoApp as J, convertDeepnoteToPercentNotebooks as K, detectFormat as L, computeSnapshotHash as M, convertIpynbFilesToDeepnoteFile as N, parseSourceFilePath as O, convertJupyterNotebookToBlocks as P, convertBlockToJupyterCell as Q, isMarimoContent as R, mergeSnapshotIntoSource as S, loadLatestSnapshot as T, serializeQuartoFormat as U, convertDeepnoteFileToQuartoFiles as V, convertBlocksToPercentNotebook as W, convertDeepnoteToMarimoApps as X, convertDeepnoteFileToMarimoFiles as Y, serializeMarimoFormat as Z, generateSnapshotFilename as _, parseQuartoFormat as a, splitDeepnoteFile as b, convertPercentNotebookToBlocks as c, readAndConvertPercentFiles as d, convertDeepnoteFileToJupyterFiles as et, convertMarimoAppToBlocks as f, readAndConvertMarimoFiles as g, parseMarimoFormat as h, convertQuartoFilesToDeepnoteFile as i, computeContentHash as j, snapshotExists as k, convertPercentNotebooksToDeepnote as l, convertMarimoFilesToDeepnoteFile as m, convertQuartoDocumentToBlocks as n, readAndConvertQuartoFiles as o, convertMarimoAppsToDeepnote as p, serializePercentFormat as q, convertQuartoDocumentsToDeepnote as r, convertPercentFilesToDeepnoteFile as s, writeDeepnoteFile as t, convertDeepnoteToJupyterNotebooks as tt, parsePercentFormat as u, hasOutputs as v, getSnapshotDir as w, countBlocksWithOutputs as x, slugifyProjectName as y, isPercentContent as z };
|