@jpmorganchase/mosaic-source-local-folder 0.1.0-beta.97 → 1.0.0-beta.99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1171 -0
- package/dist/fromFsWatch.d.ts +8 -2
- package/dist/fromFsWatch.js +85 -12
- package/dist/fromFsWatch.js.map +1 -1
- package/dist/index.d.ts +2 -19
- package/dist/index.js +157 -25
- package/dist/index.js.map +1 -1
- package/package.json +9 -11
package/dist/fromFsWatch.d.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
import { WatchOptions } from 'fs';
|
|
1
|
+
import type { WatchOptions } from 'fs';
|
|
2
2
|
import { Observable } from 'rxjs';
|
|
3
|
+
/**
|
|
4
|
+
* Emits the (relative) filename whenever something changes under `filename`.
|
|
5
|
+
*
|
|
6
|
+
* Backed by Node's built-in `fs.watch`. Falls back to manually watching every
|
|
7
|
+
* subdirectory if the platform/Node version doesn't support recursive watching.
|
|
8
|
+
*/
|
|
3
9
|
export default function fromFsWatch(filename: string, options?: WatchOptions | BufferEncoding | string | null | (WatchOptions & {
|
|
4
10
|
encoding: 'buffer';
|
|
5
|
-
}) | 'buffer'): Observable<
|
|
11
|
+
}) | 'buffer'): Observable<string>;
|
package/dist/fromFsWatch.js
CHANGED
|
@@ -1,19 +1,92 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
1
3
|
import { Observable } from 'rxjs';
|
|
2
|
-
|
|
4
|
+
/**
|
|
5
|
+
* Emits the (relative) filename whenever something changes under `filename`.
|
|
6
|
+
*
|
|
7
|
+
* Backed by Node's built-in `fs.watch`. Falls back to manually watching every
|
|
8
|
+
* subdirectory if the platform/Node version doesn't support recursive watching.
|
|
9
|
+
*/
|
|
3
10
|
export default function fromFsWatch(filename, options) {
|
|
11
|
+
const normalizedOptions = options && typeof options === 'object' ? options : undefined;
|
|
12
|
+
const wantsRecursive = !!normalizedOptions?.recursive;
|
|
4
13
|
return new Observable(subscriber => {
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
subscriber.
|
|
14
|
-
|
|
14
|
+
const watchers = [];
|
|
15
|
+
let unsubscribed = false;
|
|
16
|
+
const attach = (target, opts) => {
|
|
17
|
+
const watcher = fs.watch(target, (opts ?? {}));
|
|
18
|
+
watcher.on('change', (_eventType, changed) => {
|
|
19
|
+
const name = typeof changed === 'string' ? changed : changed?.toString() ?? '';
|
|
20
|
+
subscriber.next(name);
|
|
21
|
+
});
|
|
22
|
+
watcher.on('error', err => subscriber.error(err));
|
|
23
|
+
watchers.push(watcher);
|
|
24
|
+
return watcher;
|
|
25
|
+
};
|
|
26
|
+
try {
|
|
27
|
+
if (!wantsRecursive) {
|
|
28
|
+
attach(filename, normalizedOptions);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
try {
|
|
32
|
+
// Try native recursive watch first (macOS/Windows always, Linux >= 20).
|
|
33
|
+
attach(filename, normalizedOptions);
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
const code = err?.code;
|
|
37
|
+
if (code === 'ERR_FEATURE_UNAVAILABLE_ON_PLATFORM' || code === 'ENOSYS') {
|
|
38
|
+
// Manual recursive fallback.
|
|
39
|
+
const watchedDirs = new Set();
|
|
40
|
+
const watchDir = (dir) => {
|
|
41
|
+
if (watchedDirs.has(dir) || unsubscribed)
|
|
42
|
+
return;
|
|
43
|
+
watchedDirs.add(dir);
|
|
44
|
+
const w = attach(dir, { ...normalizedOptions, recursive: false });
|
|
45
|
+
w.on('change', (_eventType, changed) => {
|
|
46
|
+
const child = typeof changed === 'string' ? changed : changed?.toString();
|
|
47
|
+
if (!child)
|
|
48
|
+
return;
|
|
49
|
+
const full = path.join(dir, child);
|
|
50
|
+
fs.promises
|
|
51
|
+
.stat(full)
|
|
52
|
+
.then(stat => {
|
|
53
|
+
if (stat.isDirectory())
|
|
54
|
+
watchDir(full);
|
|
55
|
+
})
|
|
56
|
+
.catch(() => {
|
|
57
|
+
/* removed entry; ignore */
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
};
|
|
61
|
+
const seedRecursive = async (dir) => {
|
|
62
|
+
if (unsubscribed)
|
|
63
|
+
return;
|
|
64
|
+
watchDir(dir);
|
|
65
|
+
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
66
|
+
await Promise.all(entries.filter(e => e.isDirectory()).map(e => seedRecursive(path.join(dir, e.name))));
|
|
67
|
+
};
|
|
68
|
+
seedRecursive(filename).catch(error => subscriber.error(error));
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
throw err;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
subscriber.error(err);
|
|
78
|
+
}
|
|
15
79
|
return function unsubscribe() {
|
|
16
|
-
|
|
80
|
+
unsubscribed = true;
|
|
81
|
+
for (const w of watchers) {
|
|
82
|
+
try {
|
|
83
|
+
w.close();
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
/* ignore */
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
subscriber.complete();
|
|
17
90
|
};
|
|
18
91
|
});
|
|
19
92
|
}
|
package/dist/fromFsWatch.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fromFsWatch.js","sourceRoot":"","sources":["../src/fromFsWatch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fromFsWatch.js","sourceRoot":"","sources":["../src/fromFsWatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAElC;;;;;GAKG;AACH,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,QAAgB,EAChB,OAMY;IAEZ,MAAM,iBAAiB,GACrB,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAE,OAAwB,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,MAAM,cAAc,GAAG,CAAC,CAAC,iBAAiB,EAAE,SAAS,CAAC;IAEtD,OAAO,IAAI,UAAU,CAAS,UAAU,CAAC,EAAE;QACzC,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,MAAM,MAAM,GAAG,CAAC,MAAc,EAAE,IAAmB,EAAE,EAAE;YACrD,MAAM,OAAO,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAmC,CAAC,CAAC;YACjF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE;gBAC3C,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;gBAC/E,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAClD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,wEAAwE;oBACxE,MAAM,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;gBACtC,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,MAAM,IAAI,GAAI,GAA6B,EAAE,IAAI,CAAC;oBAClD,IAAI,IAAI,KAAK,qCAAqC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACxE,6BAA6B;wBAC7B,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;wBACtC,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;4BAC/B,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,YAAY;gCAAE,OAAO;4BACjD,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;4BACrB,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,GAAG,iBAAiB,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;4BAClE,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE;gCACrC,MAAM,KAAK,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC;gCAC1E,IAAI,CAAC,KAAK;oCAAE,OAAO;gCACnB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gCACnC,EAAE,CAAC,QAAQ;qCACR,IAAI,CAAC,IAAI,CAAC;qCACV,IAAI,CAAC,IAAI,CAAC,EAAE;oCACX,IAAI,IAAI,CAAC,WAAW,EAAE;wCAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;gCACzC,CAAC,CAAC;qCACD,KAAK,CAAC,GAAG,EAAE;oCACV,2BAA2B;gCAC7B,CAAC,CAAC,CAAC;4BACP,CAAC,CAAC,CAAC;wBACL,CAAC,CAAC;wBAEF,MAAM,aAAa,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;4BAC1C,IAAI,YAAY;gCAAE,OAAO;4BACzB,QAAQ,CAAC,GAAG,CAAC,CAAC;4BACd,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;4BACxE,MAAM,OAAO,CAAC,GAAG,CACf,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CACrF,CAAC;wBACJ,CAAC,CAAC;wBAEF,aAAa,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;oBAClE,CAAC;yBAAM,CAAC;wBACN,MAAM,GAAG,CAAC;oBACZ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,CAAC,KAAK,CAAC,GAAY,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,SAAS,WAAW;YACzB,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,CAAC,CAAC,KAAK,EAAE,CAAC;gBACZ,CAAC;gBAAC,MAAM,CAAC;oBACP,YAAY;gBACd,CAAC;YACH,CAAC;YACD,UAAU,CAAC,QAAQ,EAAE,CAAC;QACxB,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,27 +1,10 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import type { Source } from '@jpmorganchase/mosaic-types';
|
|
3
3
|
export declare const schema: z.ZodObject<{
|
|
4
|
-
|
|
5
|
-
* Collection of file extensions to look for
|
|
6
|
-
*/
|
|
7
|
-
extensions: z.ZodArray<z.ZodString, "atleastone">;
|
|
8
|
-
/**
|
|
9
|
-
* Add to use a folder prefix
|
|
10
|
-
*/
|
|
4
|
+
extensions: z.ZodTuple<[z.ZodString], z.ZodString>;
|
|
11
5
|
prefixDir: z.ZodOptional<z.ZodString>;
|
|
12
|
-
/**
|
|
13
|
-
* The root directory containing docs
|
|
14
|
-
*/
|
|
15
6
|
rootDir: z.ZodString;
|
|
16
|
-
},
|
|
17
|
-
extensions: [string, ...string[]];
|
|
18
|
-
rootDir: string;
|
|
19
|
-
prefixDir?: string | undefined;
|
|
20
|
-
}, {
|
|
21
|
-
extensions: [string, ...string[]];
|
|
22
|
-
rootDir: string;
|
|
23
|
-
prefixDir?: string | undefined;
|
|
24
|
-
}>;
|
|
7
|
+
}, z.core.$strip>;
|
|
25
8
|
export type LocalFolderSourceOptions = z.infer<typeof schema>;
|
|
26
9
|
declare const LocalFolderSource: Source<LocalFolderSourceOptions>;
|
|
27
10
|
export default LocalFolderSource;
|
package/dist/index.js
CHANGED
|
@@ -1,30 +1,66 @@
|
|
|
1
|
-
import { concatMap,
|
|
2
|
-
import glob from '
|
|
1
|
+
import { concatMap, debounceTime, merge, of, switchMap } from 'rxjs';
|
|
2
|
+
import { glob } from 'tinyglobby';
|
|
3
3
|
import path from 'path';
|
|
4
4
|
import fs from 'fs';
|
|
5
|
-
import { merge as lodashMerge } from 'lodash-es';
|
|
6
5
|
import { z } from 'zod';
|
|
7
6
|
import { validateMosaicSchema } from '@jpmorganchase/mosaic-schemas';
|
|
8
7
|
import fromFsWatch from './fromFsWatch.js';
|
|
9
|
-
async function getLastModifiedDate(fullPath) {
|
|
10
|
-
const resolvedRoute = await fs.promises.realpath(fullPath);
|
|
11
|
-
const stats = await fs.promises.stat(resolvedRoute);
|
|
12
|
-
return stats?.mtimeMs ? stats.mtimeMs : 0;
|
|
13
|
-
}
|
|
14
8
|
function createFileGlob(url, pageExtensions) {
|
|
9
|
+
// `**.mdx` only matches top-level files in tinyglobby; fast-glob treated it as `**/*.mdx`.
|
|
10
|
+
const base = /^(\.\/)?\*\*$/.test(url) ? `${url}/*` : url;
|
|
15
11
|
if (pageExtensions.length === 1) {
|
|
16
|
-
return `${
|
|
12
|
+
return `${base}${pageExtensions[0]}`;
|
|
17
13
|
}
|
|
18
|
-
return `${
|
|
14
|
+
return `${base}{${pageExtensions.join(',')}}`;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Build the canonical `fullPath` (URL-shaped, leading slash, optional
|
|
18
|
+
* namespace prefix) once per filepath. Pulled out only for clarity —
|
|
19
|
+
* the original computation lived inline inside the per-page builder.
|
|
20
|
+
*/
|
|
21
|
+
function buildFullPath(filepath, prefixDir) {
|
|
22
|
+
const joined = prefixDir ? path.posix.join(prefixDir, filepath) : filepath;
|
|
23
|
+
return `/${joined}`.replace(/^\/{2,}/, '/');
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Run `worker` over `items` with an upper bound on in-flight promises.
|
|
27
|
+
*
|
|
28
|
+
* `Promise.all(items.map(worker))` over a very large docs tree
|
|
29
|
+
* (thousands of MDX files) holds all parsed `Page` objects in memory
|
|
30
|
+
* simultaneously, because `Promise.all` resolves all results together
|
|
31
|
+
* before the next pipeline step can drain them. The libuv thread pool
|
|
32
|
+
* caps the *concurrent* fs reads at 4 by default, but JS micro-tasks
|
|
33
|
+
* drain bursty — when 4 reads complete simultaneously, 4 synchronous
|
|
34
|
+
* deserialise steps run back-to-back. Bounding the queue here keeps
|
|
35
|
+
* peak Page accumulation predictable on giant trees without measurably
|
|
36
|
+
* changing throughput on typical Mosaic sites (~hundreds of files),
|
|
37
|
+
* where the bound is never reached.
|
|
38
|
+
*
|
|
39
|
+
* Insurance, not a fix for any observed Mosaic-site regression — keep
|
|
40
|
+
* it if you might consume monorepo-scale docs trees, drop it if the
|
|
41
|
+
* extra ~20 lines aren't worth the future-proofing.
|
|
42
|
+
*/
|
|
43
|
+
async function mapWithConcurrency(items, limit, worker) {
|
|
44
|
+
const results = new Array(items.length);
|
|
45
|
+
let cursor = 0;
|
|
46
|
+
const runners = new Array(Math.min(limit, items.length)).fill(0).map(async () => {
|
|
47
|
+
for (;;) {
|
|
48
|
+
const i = cursor++;
|
|
49
|
+
if (i >= items.length)
|
|
50
|
+
return;
|
|
51
|
+
results[i] = await worker(items[i], i);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
await Promise.all(runners);
|
|
55
|
+
return results;
|
|
19
56
|
}
|
|
57
|
+
const READ_CONCURRENCY = 16;
|
|
58
|
+
const STAT_CONCURRENCY = 64;
|
|
20
59
|
export const schema = z.object({
|
|
21
60
|
/**
|
|
22
61
|
* Collection of file extensions to look for
|
|
23
62
|
*/
|
|
24
|
-
extensions: z
|
|
25
|
-
.string({ required_error: 'Please provide the collection of file extensions to look for' })
|
|
26
|
-
.array()
|
|
27
|
-
.nonempty(),
|
|
63
|
+
extensions: z.tuple([z.string()], z.string()),
|
|
28
64
|
/**
|
|
29
65
|
* Add to use a folder prefix
|
|
30
66
|
*/
|
|
@@ -32,23 +68,119 @@ export const schema = z.object({
|
|
|
32
68
|
/**
|
|
33
69
|
* The root directory containing docs
|
|
34
70
|
*/
|
|
35
|
-
rootDir: z.string({
|
|
71
|
+
rootDir: z.string({
|
|
72
|
+
error: issue => (issue.input === undefined ? 'Please provide a root directory name' : undefined)
|
|
73
|
+
})
|
|
36
74
|
});
|
|
37
75
|
const LocalFolderSource = {
|
|
76
|
+
/**
|
|
77
|
+
* WARNING for upstream consumers: the per-file deserialise cache
|
|
78
|
+
* (`cache` below) is scoped to a single `create()` invocation. If you
|
|
79
|
+
* compose this source under an operator that re-subscribes the
|
|
80
|
+
* returned observable on every tick of a polling stream (e.g.
|
|
81
|
+
* `switchMap(() => watchFolder$)`, `exhaustMap`, …), the cache is
|
|
82
|
+
* thrown away and rebuilt from scratch on every re-subscription,
|
|
83
|
+
* producing the worst possible memory churn and GC pressure on large
|
|
84
|
+
* docs trees. Subscribe once for the lifetime of the source and
|
|
85
|
+
* merge in side-streams as *signals*, not triggers.
|
|
86
|
+
*/
|
|
38
87
|
create(options, { serialiser }) {
|
|
39
88
|
validateMosaicSchema(schema, options);
|
|
40
|
-
|
|
89
|
+
// Per-`create()` cache, keyed by the relative filepath returned
|
|
90
|
+
// from `glob`. Lives for the lifetime of the Observable's
|
|
91
|
+
// subscription; each `LocalFolderSource` instance gets its own,
|
|
92
|
+
// so it composes cleanly with `prefixDir` / multi-source setups.
|
|
93
|
+
const cache = new Map();
|
|
94
|
+
// Files whose real location is outside `rootDir` (via a symlinked file
|
|
95
|
+
// or folder). Warn once each rather than on every re-scan.
|
|
96
|
+
const reportedOutsideRoot = new Set();
|
|
97
|
+
return merge(of(null), fromFsWatch(options.rootDir, { recursive: true })).pipe(
|
|
98
|
+
// Coalesce the burst of `rename`/`change` events most editors emit
|
|
99
|
+
// for a single save (atomic write → rename + chmod). 50 ms is below
|
|
100
|
+
// the human-perception threshold while comfortably wider than any
|
|
101
|
+
// editor's write burst.
|
|
102
|
+
debounceTime(50), switchMap(() => glob(createFileGlob('**', options.extensions), {
|
|
41
103
|
cwd: options.rootDir,
|
|
42
|
-
onlyFiles: true
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
104
|
+
onlyFiles: true,
|
|
105
|
+
expandDirectories: false
|
|
106
|
+
})), concatMap(async (filepaths) => {
|
|
107
|
+
// Symlinks are followed by the glob, so a link committed to the
|
|
108
|
+
// content folder could point at any file the process can read
|
|
109
|
+
// (e.g. `leak.mdx -> /proc/self/environ`). Only publish files
|
|
110
|
+
// whose real path is inside the real root.
|
|
111
|
+
const realRoot = await fs.promises.realpath(options.rootDir);
|
|
112
|
+
// Stage 1: stat every file. `stat` is metadata-only (no read)
|
|
113
|
+
// and libuv caps the actual concurrent stats at its thread-pool
|
|
114
|
+
// size (4 by default), so the `STAT_CONCURRENCY` ceiling is
|
|
115
|
+
// mostly there to keep the in-memory queue of pending closures
|
|
116
|
+
// bounded on giant trees rather than to control I/O parallelism.
|
|
117
|
+
// Files that vanished between `glob` and `stat` are tolerated
|
|
118
|
+
// by treating the entry as "skip"; the next emission picks up
|
|
119
|
+
// the steady state.
|
|
120
|
+
const stats = await mapWithConcurrency(filepaths, STAT_CONCURRENCY, async (filepath) => {
|
|
121
|
+
const fullPath = path.posix.join(options.rootDir, filepath);
|
|
122
|
+
try {
|
|
123
|
+
const realPath = await fs.promises.realpath(fullPath);
|
|
124
|
+
const relativeToRoot = path.relative(realRoot, realPath);
|
|
125
|
+
if (relativeToRoot === '..' ||
|
|
126
|
+
relativeToRoot.startsWith(`..${path.sep}`) ||
|
|
127
|
+
path.isAbsolute(relativeToRoot)) {
|
|
128
|
+
if (!reportedOutsideRoot.has(filepath)) {
|
|
129
|
+
reportedOutsideRoot.add(filepath);
|
|
130
|
+
console.warn(`[Mosaic][Source-Local-Folder] Skipping ${fullPath}: it resolves outside ${options.rootDir}.`);
|
|
131
|
+
}
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
const s = await fs.promises.stat(realPath);
|
|
135
|
+
return { filepath, fullPath, realPath, mtimeMs: s.mtimeMs, size: s.size };
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
// Stage 2: reuse the cached `Page` when `(mtimeMs, size)` are
|
|
142
|
+
// unchanged, otherwise do the full read + deserialise. The
|
|
143
|
+
// expensive work is skipped for every unchanged file —
|
|
144
|
+
// typically every file but the one the user just saved.
|
|
145
|
+
//
|
|
146
|
+
// `structuredClone` rather than `lodash.merge({}, …)`: see the
|
|
147
|
+
// `CacheEntry` comment for the rationale.
|
|
148
|
+
const pages = await mapWithConcurrency(stats, READ_CONCURRENCY, async (entry) => {
|
|
149
|
+
if (!entry)
|
|
150
|
+
return undefined;
|
|
151
|
+
const { filepath, fullPath, realPath, mtimeMs, size } = entry;
|
|
152
|
+
const cached = cache.get(filepath);
|
|
153
|
+
if (cached && cached.mtimeMs === mtimeMs && cached.size === size) {
|
|
154
|
+
return structuredClone(cached.page);
|
|
155
|
+
}
|
|
156
|
+
const data = await fs.promises.readFile(realPath);
|
|
157
|
+
const deserialised = (await serialiser.deserialise(fullPath, data));
|
|
158
|
+
// The cache holds the canonical (pre-clone) page; consumers
|
|
159
|
+
// always receive a `structuredClone` of it on every emission.
|
|
160
|
+
//
|
|
161
|
+
// `lastModified` is typed as `Date` upstream but we assign a
|
|
162
|
+
// number (`mtimeMs`) — `lodash.merge` accepted this silently
|
|
163
|
+
// and the rest of the pipeline downstream depends on the
|
|
164
|
+
// number shape, so we preserve it via an explicit cast
|
|
165
|
+
// rather than quietly fixing the type and breaking consumers.
|
|
166
|
+
const page = {
|
|
167
|
+
...deserialised,
|
|
168
|
+
lastModified: mtimeMs,
|
|
169
|
+
fullPath: buildFullPath(filepath, options.prefixDir)
|
|
170
|
+
};
|
|
171
|
+
cache.set(filepath, { mtimeMs, size, page });
|
|
172
|
+
return structuredClone(page);
|
|
50
173
|
});
|
|
51
|
-
|
|
174
|
+
// Stage 3: prune entries for files that disappeared between
|
|
175
|
+
// emissions, so the cache doesn't grow unboundedly across a
|
|
176
|
+
// long dev session.
|
|
177
|
+
const currentSet = new Set(filepaths);
|
|
178
|
+
for (const key of cache.keys()) {
|
|
179
|
+
if (!currentSet.has(key))
|
|
180
|
+
cache.delete(key);
|
|
181
|
+
}
|
|
182
|
+
return pages.filter((p) => p !== undefined);
|
|
183
|
+
}));
|
|
52
184
|
}
|
|
53
185
|
};
|
|
54
186
|
export default LocalFolderSource;
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAc,EAAE,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjF,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAErE,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C,SAAS,cAAc,CAAC,GAAW,EAAE,cAAwB;IAC3D,2FAA2F;IAC3F,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC1D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,GAAG,IAAI,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAChD,CAAC;AA8BD;;;;GAIG;AACH,SAAS,aAAa,CAAC,QAAgB,EAAE,SAA6B;IACpE,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC3E,OAAO,IAAI,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,kBAAkB,CAC/B,KAAmB,EACnB,KAAa,EACb,MAA8C;IAE9C,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;QAC9E,SAAS,CAAC;YACR,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;YACnB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAC5B,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B;;OAEG;IACH,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IAC7C;;OAEG;IACH,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC,CAAC,SAAS,CAAC;KACjG,CAAC;CACH,CAAC,CAAC;AAIH,MAAM,iBAAiB,GAAqC;IAC1D;;;;;;;;;;OAUG;IACH,MAAM,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE;QAC5B,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAEtC,gEAAgE;QAChE,0DAA0D;QAC1D,gEAAgE;QAChE,iEAAiE;QACjE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC5C,uEAAuE;QACvE,2DAA2D;QAC3D,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE9C,OAAO,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;QAC5E,mEAAmE;QACnE,oEAAoE;QACpE,kEAAkE;QAClE,wBAAwB;QACxB,YAAY,CAAC,EAAE,CAAC,EAChB,SAAS,CAAC,GAAG,EAAE,CACb,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE;YAC7C,GAAG,EAAE,OAAO,CAAC,OAAO;YACpB,SAAS,EAAE,IAAI;YACf,iBAAiB,EAAE,KAAK;SACzB,CAAC,CACH,EACD,SAAS,CAAC,KAAK,EAAE,SAAmB,EAAE,EAAE;YACtC,gEAAgE;YAChE,8DAA8D;YAC9D,8DAA8D;YAC9D,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAE7D,8DAA8D;YAC9D,gEAAgE;YAChE,4DAA4D;YAC5D,+DAA+D;YAC/D,iEAAiE;YACjE,8DAA8D;YAC9D,8DAA8D;YAC9D,oBAAoB;YACpB,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,SAAS,EAAE,gBAAgB,EAAE,KAAK,EAAC,QAAQ,EAAC,EAAE;gBACnF,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC5D,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBACtD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBACzD,IACE,cAAc,KAAK,IAAI;wBACvB,cAAc,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC1C,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,EAC/B,CAAC;wBACD,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;4BACvC,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;4BAClC,OAAO,CAAC,IAAI,CACV,0CAA0C,QAAQ,yBAAyB,OAAO,CAAC,OAAO,GAAG,CAC9F,CAAC;wBACJ,CAAC;wBACD,OAAO,SAAS,CAAC;oBACnB,CAAC;oBACD,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC3C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5E,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,SAAS,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,2DAA2D;YAC3D,uDAAuD;YACvD,wDAAwD;YACxD,EAAE;YACF,+DAA+D;YAC/D,0CAA0C;YAC1C,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;gBAC5E,IAAI,CAAC,KAAK;oBAAE,OAAO,SAAS,CAAC;gBAC7B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;gBAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACnC,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBACjE,OAAO,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACtC,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAClD,MAAM,YAAY,GAAG,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAS,CAAC;gBAC5E,4DAA4D;gBAC5D,8DAA8D;gBAC9D,EAAE;gBACF,6DAA6D;gBAC7D,6DAA6D;gBAC7D,yDAAyD;gBACzD,uDAAuD;gBACvD,8DAA8D;gBAC9D,MAAM,IAAI,GAAG;oBACX,GAAG,YAAY;oBACf,YAAY,EAAE,OAAO;oBACrB,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC;iBAClC,CAAC;gBACrB,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7C,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,4DAA4D;YAC5D,4DAA4D;YAC5D,oBAAoB;YACpB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YACtC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9C,CAAC;YAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAa,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QACzD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,eAAe,iBAAiB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jpmorganchase/mosaic-source-local-folder",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "1.0.0-beta.99",
|
|
4
4
|
"author": "",
|
|
5
5
|
"description": "Mosaic local folder source for core file system",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
}
|
|
17
17
|
},
|
|
18
18
|
"engines": {
|
|
19
|
-
"node": ">=
|
|
19
|
+
"node": ">=24"
|
|
20
20
|
},
|
|
21
21
|
"files": [
|
|
22
22
|
"dist"
|
|
@@ -24,20 +24,18 @@
|
|
|
24
24
|
"scripts": {
|
|
25
25
|
"build": "tsc",
|
|
26
26
|
"clean": "rm -fr dist",
|
|
27
|
-
"lint": "eslint
|
|
27
|
+
"lint": "eslint ."
|
|
28
28
|
},
|
|
29
29
|
"repository": {
|
|
30
30
|
"type": "git",
|
|
31
|
-
"url": "git
|
|
31
|
+
"url": "git+https://github.com/jpmorganchase/mosaic.git",
|
|
32
32
|
"directory": "packages/source-local-folder"
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@jpmorganchase/mosaic-schemas": "^
|
|
36
|
-
"@jpmorganchase/mosaic-types": "^
|
|
37
|
-
"chokidar": "^3.5.3",
|
|
38
|
-
"fast-glob": "^3.2.12",
|
|
39
|
-
"lodash-es": "^4.17.21",
|
|
35
|
+
"@jpmorganchase/mosaic-schemas": "^1.0.0-beta.99",
|
|
36
|
+
"@jpmorganchase/mosaic-types": "^1.0.0-beta.99",
|
|
40
37
|
"rxjs": "^7.5.5",
|
|
41
|
-
"
|
|
38
|
+
"tinyglobby": "^0.2.17",
|
|
39
|
+
"zod": "^4.6.7"
|
|
42
40
|
}
|
|
43
|
-
}
|
|
41
|
+
}
|