@fluid-tools/fetch-tool 2.74.0-370705 → 2.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/fluidAnalyzeMessages.d.ts +2 -2
- package/dist/fluidAnalyzeMessages.d.ts.map +1 -1
- package/dist/fluidAnalyzeMessages.js +1 -2
- package/dist/fluidAnalyzeMessages.js.map +1 -1
- package/dist/fluidFetch.js.map +1 -1
- package/dist/fluidFetchArgs.d.ts.map +1 -1
- package/dist/fluidFetchArgs.js.map +1 -1
- package/dist/fluidFetchInit.d.ts +2 -1
- package/dist/fluidFetchInit.d.ts.map +1 -1
- package/dist/fluidFetchInit.js +1 -1
- package/dist/fluidFetchInit.js.map +1 -1
- package/dist/fluidFetchMessages.d.ts.map +1 -1
- package/dist/fluidFetchMessages.js +1 -2
- package/dist/fluidFetchMessages.js.map +1 -1
- package/dist/fluidFetchSharePoint.d.ts.map +1 -1
- package/dist/fluidFetchSharePoint.js.map +1 -1
- package/dist/fluidFetchSnapshot.d.ts.map +1 -1
- package/dist/fluidFetchSnapshot.js.map +1 -1
- package/eslint.config.mts +42 -0
- package/lib/fluidAnalyzeMessages.d.ts +2 -2
- package/lib/fluidAnalyzeMessages.d.ts.map +1 -1
- package/lib/fluidAnalyzeMessages.js +1 -2
- package/lib/fluidAnalyzeMessages.js.map +1 -1
- package/lib/fluidFetch.js.map +1 -1
- package/lib/fluidFetchArgs.d.ts.map +1 -1
- package/lib/fluidFetchArgs.js.map +1 -1
- package/lib/fluidFetchInit.d.ts +2 -1
- package/lib/fluidFetchInit.d.ts.map +1 -1
- package/lib/fluidFetchInit.js +1 -1
- package/lib/fluidFetchInit.js.map +1 -1
- package/lib/fluidFetchMessages.d.ts.map +1 -1
- package/lib/fluidFetchMessages.js +1 -2
- package/lib/fluidFetchMessages.js.map +1 -1
- package/lib/fluidFetchSharePoint.d.ts.map +1 -1
- package/lib/fluidFetchSharePoint.js.map +1 -1
- package/lib/fluidFetchSnapshot.d.ts.map +1 -1
- package/lib/fluidFetchSnapshot.js.map +1 -1
- package/package.json +22 -21
- package/src/fluidAnalyzeMessages.ts +13 -13
- package/src/fluidFetch.ts +7 -4
- package/src/fluidFetchArgs.ts +6 -6
- package/src/fluidFetchInit.ts +12 -7
- package/src/fluidFetchMessages.ts +4 -4
- package/src/fluidFetchSharePoint.ts +8 -4
- package/src/fluidFetchSnapshot.ts +17 -7
package/src/fluidFetchArgs.ts
CHANGED
|
@@ -47,7 +47,7 @@ const optionsArray = [
|
|
|
47
47
|
["--loginHint", "login hint for the user with document access."],
|
|
48
48
|
];
|
|
49
49
|
|
|
50
|
-
function printUsage() {
|
|
50
|
+
function printUsage(): void {
|
|
51
51
|
console.log("Usage: fluid-fetch [options] URL");
|
|
52
52
|
console.log("URL: <ODSP URL>|<Routerlicious URL>");
|
|
53
53
|
console.log("Options:");
|
|
@@ -66,7 +66,7 @@ export function setArguments(values: {
|
|
|
66
66
|
dumpSnapshotStats?: boolean;
|
|
67
67
|
dumpSnapshotTrees?: boolean;
|
|
68
68
|
overWrite?: boolean;
|
|
69
|
-
}) {
|
|
69
|
+
}): void {
|
|
70
70
|
paramSaveDir = values.saveDir;
|
|
71
71
|
paramURL = values.paramURL;
|
|
72
72
|
dumpMessages = values.dumpMessages ?? dumpMessages;
|
|
@@ -76,7 +76,7 @@ export function setArguments(values: {
|
|
|
76
76
|
overWrite = values.overWrite ?? overWrite;
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
-
export function parseArguments() {
|
|
79
|
+
export function parseArguments(): void {
|
|
80
80
|
for (let i = 2; i < process.argv.length; i++) {
|
|
81
81
|
const arg = process.argv[i];
|
|
82
82
|
switch (arg) {
|
|
@@ -157,7 +157,7 @@ export function parseArguments() {
|
|
|
157
157
|
checkArgs();
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
-
function parseStrArg(i: number, name: string) {
|
|
160
|
+
function parseStrArg(i: number, name: string): string {
|
|
161
161
|
if (i + 1 >= process.argv.length) {
|
|
162
162
|
console.error(`ERROR: Missing ${name}`);
|
|
163
163
|
printUsage();
|
|
@@ -165,7 +165,7 @@ function parseStrArg(i: number, name: string) {
|
|
|
165
165
|
}
|
|
166
166
|
return process.argv[i + 1];
|
|
167
167
|
}
|
|
168
|
-
function parseIntArg(i: number, name: string, allowZero: boolean) {
|
|
168
|
+
function parseIntArg(i: number, name: string, allowZero: boolean): number {
|
|
169
169
|
if (i + 1 >= process.argv.length) {
|
|
170
170
|
console.error(`ERROR: Missing ${name}`);
|
|
171
171
|
printUsage();
|
|
@@ -181,7 +181,7 @@ function parseIntArg(i: number, name: string, allowZero: boolean) {
|
|
|
181
181
|
return paramNumber;
|
|
182
182
|
}
|
|
183
183
|
|
|
184
|
-
function checkArgs() {
|
|
184
|
+
function checkArgs(): void {
|
|
185
185
|
if (paramSnapshotVersionIndex !== undefined) {
|
|
186
186
|
paramNumSnapshotVersions = Math.max(
|
|
187
187
|
paramSnapshotVersionIndex + 1,
|
package/src/fluidFetchInit.ts
CHANGED
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import type { IRequest } from "@fluidframework/core-interfaces";
|
|
7
|
-
import type {
|
|
7
|
+
import type {
|
|
8
|
+
IDocumentService,
|
|
9
|
+
IResolvedUrl,
|
|
10
|
+
} from "@fluidframework/driver-definitions/internal";
|
|
8
11
|
import type {
|
|
9
12
|
IPublicClientConfig,
|
|
10
13
|
IOdspAuthRequestInfo,
|
|
@@ -31,7 +34,7 @@ async function initializeODSPCore(
|
|
|
31
34
|
odspResolvedUrl: IOdspResolvedUrl,
|
|
32
35
|
server: string,
|
|
33
36
|
clientConfig: IPublicClientConfig,
|
|
34
|
-
) {
|
|
37
|
+
): Promise<IDocumentService | undefined> {
|
|
35
38
|
const { driveId, itemId } = odspResolvedUrl;
|
|
36
39
|
|
|
37
40
|
connectionInfo = {
|
|
@@ -52,7 +55,9 @@ async function initializeODSPCore(
|
|
|
52
55
|
item: ${itemId}
|
|
53
56
|
docId: ${docId}`);
|
|
54
57
|
|
|
55
|
-
const getStorageTokenStub = async (
|
|
58
|
+
const getStorageTokenStub = async (
|
|
59
|
+
options: OdspResourceTokenFetchOptions,
|
|
60
|
+
): Promise<string> => {
|
|
56
61
|
return resolveWrapper(
|
|
57
62
|
async (authRequestInfo: IOdspAuthRequestInfo) => {
|
|
58
63
|
if (
|
|
@@ -68,7 +73,7 @@ async function initializeODSPCore(
|
|
|
68
73
|
);
|
|
69
74
|
};
|
|
70
75
|
// eslint-disable-next-line @typescript-eslint/promise-function-async
|
|
71
|
-
const getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions) =>
|
|
76
|
+
const getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions): Promise<string> =>
|
|
72
77
|
Promise.resolve("");
|
|
73
78
|
const odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(
|
|
74
79
|
getStorageTokenStub,
|
|
@@ -86,7 +91,7 @@ async function initializeR11s(
|
|
|
86
91
|
server: string,
|
|
87
92
|
pathname: string,
|
|
88
93
|
r11sResolvedUrl: IResolvedUrl,
|
|
89
|
-
) {
|
|
94
|
+
): Promise<IDocumentService | undefined> {
|
|
90
95
|
const path = pathname.split("/");
|
|
91
96
|
let tenantId: string;
|
|
92
97
|
let documentId: string;
|
|
@@ -108,7 +113,7 @@ async function initializeR11s(
|
|
|
108
113
|
};
|
|
109
114
|
|
|
110
115
|
if (localDataOnly) {
|
|
111
|
-
return;
|
|
116
|
+
return undefined;
|
|
112
117
|
}
|
|
113
118
|
|
|
114
119
|
console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);
|
|
@@ -159,7 +164,7 @@ async function resolveUrl(url: string): Promise<IResolvedInfo | undefined> {
|
|
|
159
164
|
return undefined;
|
|
160
165
|
}
|
|
161
166
|
|
|
162
|
-
export async function fluidFetchInit(urlStr: string) {
|
|
167
|
+
export async function fluidFetchInit(urlStr: string): Promise<IDocumentService | undefined> {
|
|
163
168
|
const resolvedInfo = await resolveUrl(urlStr);
|
|
164
169
|
if (resolvedInfo === undefined) {
|
|
165
170
|
throw new Error(`Unknown URL ${urlStr}`);
|
|
@@ -33,7 +33,7 @@ async function* loadAllSequencedMessages(
|
|
|
33
33
|
documentService?: IDocumentService,
|
|
34
34
|
dir?: string,
|
|
35
35
|
files?: string[],
|
|
36
|
-
) {
|
|
36
|
+
): AsyncGenerator<ISequencedDocumentMessage[]> {
|
|
37
37
|
let lastSeq = 0;
|
|
38
38
|
// flag for mismatch between last sequence number read and new one to be read
|
|
39
39
|
let seqNumMismatch = false;
|
|
@@ -210,10 +210,10 @@ async function* loadAllSequencedMessages(
|
|
|
210
210
|
}
|
|
211
211
|
|
|
212
212
|
async function* saveOps(
|
|
213
|
-
gen
|
|
213
|
+
gen: AsyncGenerator<ISequencedDocumentMessage[]>,
|
|
214
214
|
dir: string,
|
|
215
215
|
files: string[],
|
|
216
|
-
) {
|
|
216
|
+
): AsyncGenerator<ISequencedDocumentMessage[]> {
|
|
217
217
|
// Split into 100K ops
|
|
218
218
|
const chunk = 100 * 1000;
|
|
219
219
|
|
|
@@ -287,7 +287,7 @@ async function* saveOps(
|
|
|
287
287
|
export async function fluidFetchMessages(
|
|
288
288
|
documentService?: IDocumentService,
|
|
289
289
|
saveDir?: string,
|
|
290
|
-
) {
|
|
290
|
+
): Promise<void> {
|
|
291
291
|
const messageStats = dumpMessageStats || dumpMessages;
|
|
292
292
|
if (!messageStats && (saveDir === undefined || documentService === undefined)) {
|
|
293
293
|
return;
|
|
@@ -113,7 +113,7 @@ async function resolveDriveItemByServerRelativePath(
|
|
|
113
113
|
server: string,
|
|
114
114
|
serverRelativePath: string,
|
|
115
115
|
clientConfig: IPublicClientConfig,
|
|
116
|
-
) {
|
|
116
|
+
): Promise<IOdspDriveItem> {
|
|
117
117
|
return resolveWrapper<IOdspDriveItem>(
|
|
118
118
|
// eslint-disable-next-line @typescript-eslint/promise-function-async
|
|
119
119
|
(authRequestInfo) =>
|
|
@@ -127,7 +127,7 @@ async function resolveChildrenByDriveItem(
|
|
|
127
127
|
server: string,
|
|
128
128
|
folderDriveItem: IOdspDriveItem,
|
|
129
129
|
clientConfig: IPublicClientConfig,
|
|
130
|
-
) {
|
|
130
|
+
): Promise<IOdspDriveItem[]> {
|
|
131
131
|
return resolveWrapper<IOdspDriveItem[]>(
|
|
132
132
|
// eslint-disable-next-line @typescript-eslint/promise-function-async
|
|
133
133
|
(authRequestInfo) => getChildrenByDriveItem(folderDriveItem, server, authRequestInfo),
|
|
@@ -140,7 +140,7 @@ export async function getSharepointFiles(
|
|
|
140
140
|
server: string,
|
|
141
141
|
serverRelativePath: string,
|
|
142
142
|
recurse: boolean,
|
|
143
|
-
) {
|
|
143
|
+
): Promise<IOdspDriveItem[]> {
|
|
144
144
|
const fileInfo = await resolveDriveItemByServerRelativePath(
|
|
145
145
|
server,
|
|
146
146
|
serverRelativePath,
|
|
@@ -177,7 +177,11 @@ export async function getSharepointFiles(
|
|
|
177
177
|
return files;
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
-
export async function getSingleSharePointFile(
|
|
180
|
+
export async function getSingleSharePointFile(
|
|
181
|
+
server: string,
|
|
182
|
+
drive: string,
|
|
183
|
+
item: string,
|
|
184
|
+
): Promise<IOdspDriveItem> {
|
|
181
185
|
return resolveWrapper<IOdspDriveItem>(
|
|
182
186
|
// eslint-disable-next-line @typescript-eslint/promise-function-async
|
|
183
187
|
(authRequestInfo) => getDriveItemFromDriveAndItem(server, drive, item, authRequestInfo),
|
|
@@ -66,7 +66,7 @@ function fetchBlobs(
|
|
|
66
66
|
tree: ISnapshotTree,
|
|
67
67
|
storage: IDocumentStorageService,
|
|
68
68
|
blobIdMap: Map<string, number>,
|
|
69
|
-
) {
|
|
69
|
+
): IFetchedBlob[] {
|
|
70
70
|
const result: IFetchedBlob[] = [];
|
|
71
71
|
for (const [item, blobId] of Object.entries(tree.blobs)) {
|
|
72
72
|
const treePath = `${prefix}${item}`;
|
|
@@ -150,12 +150,15 @@ async function fetchBlobsFromSnapshotTree(
|
|
|
150
150
|
return result;
|
|
151
151
|
}
|
|
152
152
|
|
|
153
|
-
function getDumpFetchedData(fetchedData: IFetchedData[]) {
|
|
153
|
+
function getDumpFetchedData(fetchedData: IFetchedData[]): IFetchedData[] {
|
|
154
154
|
const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));
|
|
155
155
|
return sorted.filter((item) => !isFetchedTree(item) || !item.patched);
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
-
async function dumpSnapshotTreeVerbose(
|
|
158
|
+
async function dumpSnapshotTreeVerbose(
|
|
159
|
+
name: string,
|
|
160
|
+
fetchedData: IFetchedData[],
|
|
161
|
+
): Promise<void> {
|
|
159
162
|
let size = 0;
|
|
160
163
|
const sorted = getDumpFetchedData(fetchedData);
|
|
161
164
|
|
|
@@ -214,7 +217,11 @@ async function dumpSnapshotTree(
|
|
|
214
217
|
return { blobCountNew, blobCount: sorted.length, size, sizeNew };
|
|
215
218
|
}
|
|
216
219
|
|
|
217
|
-
async function saveSnapshot(
|
|
220
|
+
async function saveSnapshot(
|
|
221
|
+
name: string,
|
|
222
|
+
fetchedData: IFetchedData[],
|
|
223
|
+
saveDir: string,
|
|
224
|
+
): Promise<void> {
|
|
218
225
|
const outDir = `${saveDir}/${name}/`;
|
|
219
226
|
const mkdir = util.promisify(fs.mkdir);
|
|
220
227
|
|
|
@@ -255,7 +262,10 @@ async function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir:
|
|
|
255
262
|
);
|
|
256
263
|
}
|
|
257
264
|
|
|
258
|
-
async function fetchBlobsFromVersion(
|
|
265
|
+
async function fetchBlobsFromVersion(
|
|
266
|
+
storage: IDocumentStorageService,
|
|
267
|
+
version: IVersion,
|
|
268
|
+
): Promise<IFetchedData[]> {
|
|
259
269
|
const tree = await reportErrors(
|
|
260
270
|
`getSnapshotTree ${version.id}`,
|
|
261
271
|
storage.getSnapshotTree(version),
|
|
@@ -266,7 +276,7 @@ async function fetchBlobsFromVersion(storage: IDocumentStorageService, version:
|
|
|
266
276
|
return fetchBlobsFromSnapshotTree(storage, tree);
|
|
267
277
|
}
|
|
268
278
|
|
|
269
|
-
async function reportErrors<T>(message: string, res: Promise<T>) {
|
|
279
|
+
async function reportErrors<T>(message: string, res: Promise<T>): Promise<T> {
|
|
270
280
|
try {
|
|
271
281
|
return await res;
|
|
272
282
|
} catch (error) {
|
|
@@ -278,7 +288,7 @@ async function reportErrors<T>(message: string, res: Promise<T>) {
|
|
|
278
288
|
export async function fluidFetchSnapshot(
|
|
279
289
|
documentService?: IDocumentService,
|
|
280
290
|
saveDir?: string,
|
|
281
|
-
) {
|
|
291
|
+
): Promise<void> {
|
|
282
292
|
if (
|
|
283
293
|
!dumpSnapshotStats &&
|
|
284
294
|
!dumpSnapshotTrees &&
|