@jpmorganchase/mosaic-source-local-folder 0.1.0-beta.98 → 1.0.0-beta.99

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,11 @@
1
- import { WatchOptions } from 'fs';
1
+ import type { WatchOptions } from 'fs';
2
2
  import { Observable } from 'rxjs';
3
+ /**
4
+ * Emits the (relative) filename whenever something changes under `filename`.
5
+ *
6
+ * Backed by Node's built-in `fs.watch`. Falls back to manually watching every
7
+ * subdirectory if the platform/Node version doesn't support recursive watching.
8
+ */
3
9
  export default function fromFsWatch(filename: string, options?: WatchOptions | BufferEncoding | string | null | (WatchOptions & {
4
10
  encoding: 'buffer';
5
- }) | 'buffer'): Observable<unknown>;
11
+ }) | 'buffer'): Observable<string>;
@@ -1,19 +1,92 @@
1
+ import fs from 'fs';
2
+ import path from 'path';
1
3
  import { Observable } from 'rxjs';
2
- import { watch } from 'chokidar';
4
+ /**
5
+ * Emits the (relative) filename whenever something changes under `filename`.
6
+ *
7
+ * Backed by Node's built-in `fs.watch`. Falls back to manually watching every
8
+ * subdirectory if the platform/Node version doesn't support recursive watching.
9
+ */
3
10
  export default function fromFsWatch(filename, options) {
11
+ const normalizedOptions = options && typeof options === 'object' ? options : undefined;
12
+ const wantsRecursive = !!normalizedOptions?.recursive;
4
13
  return new Observable(subscriber => {
5
- const watcher = watch(filename, options);
6
- watcher.on('all', (_eventType, filename) => {
7
- subscriber.next(filename);
8
- });
9
- watcher.once('error', (error) => {
10
- subscriber.error(error);
11
- });
12
- watcher.once('close', () => {
13
- subscriber.complete();
14
- });
14
+ const watchers = [];
15
+ let unsubscribed = false;
16
+ const attach = (target, opts) => {
17
+ const watcher = fs.watch(target, (opts ?? {}));
18
+ watcher.on('change', (_eventType, changed) => {
19
+ const name = typeof changed === 'string' ? changed : changed?.toString() ?? '';
20
+ subscriber.next(name);
21
+ });
22
+ watcher.on('error', err => subscriber.error(err));
23
+ watchers.push(watcher);
24
+ return watcher;
25
+ };
26
+ try {
27
+ if (!wantsRecursive) {
28
+ attach(filename, normalizedOptions);
29
+ }
30
+ else {
31
+ try {
32
+ // Try native recursive watch first (macOS/Windows always, Linux >= 20).
33
+ attach(filename, normalizedOptions);
34
+ }
35
+ catch (err) {
36
+ const code = err?.code;
37
+ if (code === 'ERR_FEATURE_UNAVAILABLE_ON_PLATFORM' || code === 'ENOSYS') {
38
+ // Manual recursive fallback.
39
+ const watchedDirs = new Set();
40
+ const watchDir = (dir) => {
41
+ if (watchedDirs.has(dir) || unsubscribed)
42
+ return;
43
+ watchedDirs.add(dir);
44
+ const w = attach(dir, { ...normalizedOptions, recursive: false });
45
+ w.on('change', (_eventType, changed) => {
46
+ const child = typeof changed === 'string' ? changed : changed?.toString();
47
+ if (!child)
48
+ return;
49
+ const full = path.join(dir, child);
50
+ fs.promises
51
+ .stat(full)
52
+ .then(stat => {
53
+ if (stat.isDirectory())
54
+ watchDir(full);
55
+ })
56
+ .catch(() => {
57
+ /* removed entry; ignore */
58
+ });
59
+ });
60
+ };
61
+ const seedRecursive = async (dir) => {
62
+ if (unsubscribed)
63
+ return;
64
+ watchDir(dir);
65
+ const entries = await fs.promises.readdir(dir, { withFileTypes: true });
66
+ await Promise.all(entries.filter(e => e.isDirectory()).map(e => seedRecursive(path.join(dir, e.name))));
67
+ };
68
+ seedRecursive(filename).catch(error => subscriber.error(error));
69
+ }
70
+ else {
71
+ throw err;
72
+ }
73
+ }
74
+ }
75
+ }
76
+ catch (err) {
77
+ subscriber.error(err);
78
+ }
15
79
  return function unsubscribe() {
16
- watcher.close();
80
+ unsubscribed = true;
81
+ for (const w of watchers) {
82
+ try {
83
+ w.close();
84
+ }
85
+ catch {
86
+ /* ignore */
87
+ }
88
+ }
89
+ subscriber.complete();
17
90
  };
18
91
  });
19
92
  }
@@ -1 +1 @@
1
- {"version":3,"file":"fromFsWatch.js","sourceRoot":"","sources":["../src/fromFsWatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAEjC,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,QAAgB,EAChB,OAMY;IAEZ,OAAO,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE;QACjC,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,OAAc,CAAC,CAAC;QAEhD,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE;YACzC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YACrC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE;YACzB,UAAU,CAAC,QAAQ,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,OAAO,SAAS,WAAW;YACzB,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"fromFsWatch.js","sourceRoot":"","sources":["../src/fromFsWatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAElC;;;;;GAKG;AACH,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,QAAgB,EAChB,OAMY;IAEZ,MAAM,iBAAiB,GACrB,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAE,OAAwB,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,MAAM,cAAc,GAAG,CAAC,CAAC,iBAAiB,EAAE,SAAS,CAAC;IAEtD,OAAO,IAAI,UAAU,CAAS,UAAU,CAAC,EAAE;QACzC,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,MAAM,MAAM,GAAG,CAAC,MAAc,EAAE,IAAmB,EAAE,EAAE;YACrD,MAAM,OAAO,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAmC,CAAC,CAAC;YACjF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE;gBAC3C,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;gBAC/E,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAClD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,wEAAwE;oBACxE,MAAM,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;gBACtC,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,MAAM,IAAI,GAAI,GAA6B,EAAE,IAAI,CAAC;oBAClD,IAAI,IAAI,KAAK,qCAAqC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACxE,6BAA6B;wBAC7B,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;wBACtC,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;4BAC/B,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,YAAY;gCAAE,OAAO;4BACjD,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;4BACrB,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,GAAG,iBAAiB,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;4BAClE,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE;gCACrC,MAAM,KAAK,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC;gCAC1E,IAAI,CAAC,KAAK;oCAAE,OAAO;gCACnB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gCACnC,EAAE,CAAC,QAAQ;qCACR,IAAI,CAAC,IAAI,CAAC;qCACV,IAAI,CAAC,IAAI,CAAC,EAAE;oCACX,IAAI,IAAI,CAAC,WAAW,EAAE;wCAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;gCACzC,CAAC,CAAC;qCACD,KAAK,CAAC,GAAG,EAAE;oCACV,2BAA2B;gCAC7B,CAAC,CAAC,CAAC;4BACP,CAAC,CAAC,CAAC;wBACL,CAAC,CAAC;wBAEF,MAAM,aAAa,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;4BAC1C,IAAI,YAAY;gCAAE,OAAO;4BACzB,QAAQ,CAAC,GAAG,CAAC,CAAC;4BACd,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;4BACxE,MAAM,OAAO,CAAC,GAAG,CACf,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CACrF,CAAC;wBACJ,CAAC,CAAC;wBAEF,aAAa,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;oBAClE,CAAC;yBAAM,CAAC;wBACN,MAAM,GAAG,CAAC;oBACZ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,CAAC,KAAK,CAAC,GAAY,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,SAAS,WAAW;YACzB,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,CAAC,CAAC,KAAK,EAAE,CAAC;gBACZ,CAAC;gBAAC,MAAM,CAAC;oBACP,YAAY;gBACd,CAAC;YACH,CAAC;YACD,UAAU,CAAC,QAAQ,EAAE,CAAC;QACxB,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
package/dist/index.d.ts CHANGED
@@ -1,27 +1,10 @@
1
1
  import { z } from 'zod';
2
2
  import type { Source } from '@jpmorganchase/mosaic-types';
3
3
  export declare const schema: z.ZodObject<{
4
- /**
5
- * Collection of file extensions to look for
6
- */
7
- extensions: z.ZodArray<z.ZodString, "atleastone">;
8
- /**
9
- * Add to use a folder prefix
10
- */
4
+ extensions: z.ZodTuple<[z.ZodString], z.ZodString>;
11
5
  prefixDir: z.ZodOptional<z.ZodString>;
12
- /**
13
- * The root directory containing docs
14
- */
15
6
  rootDir: z.ZodString;
16
- }, "strip", z.ZodTypeAny, {
17
- extensions: [string, ...string[]];
18
- rootDir: string;
19
- prefixDir?: string | undefined;
20
- }, {
21
- extensions: [string, ...string[]];
22
- rootDir: string;
23
- prefixDir?: string | undefined;
24
- }>;
7
+ }, z.core.$strip>;
25
8
  export type LocalFolderSourceOptions = z.infer<typeof schema>;
26
9
  declare const LocalFolderSource: Source<LocalFolderSourceOptions>;
27
10
  export default LocalFolderSource;
package/dist/index.js CHANGED
@@ -1,30 +1,66 @@
1
- import { concatMap, delay, merge, of, switchMap } from 'rxjs';
2
- import glob from 'fast-glob';
1
+ import { concatMap, debounceTime, merge, of, switchMap } from 'rxjs';
2
+ import { glob } from 'tinyglobby';
3
3
  import path from 'path';
4
4
  import fs from 'fs';
5
- import { merge as lodashMerge } from 'lodash-es';
6
5
  import { z } from 'zod';
7
6
  import { validateMosaicSchema } from '@jpmorganchase/mosaic-schemas';
8
7
  import fromFsWatch from './fromFsWatch.js';
9
- async function getLastModifiedDate(fullPath) {
10
- const resolvedRoute = await fs.promises.realpath(fullPath);
11
- const stats = await fs.promises.stat(resolvedRoute);
12
- return stats?.mtimeMs ? stats.mtimeMs : 0;
13
- }
14
8
  function createFileGlob(url, pageExtensions) {
9
+ // `**.mdx` only matches top-level files in tinyglobby; fast-glob treated it as `**/*.mdx`.
10
+ const base = /^(\.\/)?\*\*$/.test(url) ? `${url}/*` : url;
15
11
  if (pageExtensions.length === 1) {
16
- return `${url}${pageExtensions[0]}`;
12
+ return `${base}${pageExtensions[0]}`;
17
13
  }
18
- return `${url}{${pageExtensions.join(',')}}`;
14
+ return `${base}{${pageExtensions.join(',')}}`;
15
+ }
16
+ /**
17
+ * Build the canonical `fullPath` (URL-shaped, leading slash, optional
18
+ * namespace prefix) once per filepath. Pulled out only for clarity —
19
+ * the original computation lived inline inside the per-page builder.
20
+ */
21
+ function buildFullPath(filepath, prefixDir) {
22
+ const joined = prefixDir ? path.posix.join(prefixDir, filepath) : filepath;
23
+ return `/${joined}`.replace(/^\/{2,}/, '/');
24
+ }
25
+ /**
26
+ * Run `worker` over `items` with an upper bound on in-flight promises.
27
+ *
28
+ * `Promise.all(items.map(worker))` over a very large docs tree
29
+ * (thousands of MDX files) holds all parsed `Page` objects in memory
30
+ * simultaneously, because `Promise.all` resolves all results together
31
+ * before the next pipeline step can drain them. The libuv thread pool
32
+ * caps the *concurrent* fs reads at 4 by default, but JS micro-tasks
33
+ * drain bursty — when 4 reads complete simultaneously, 4 synchronous
34
+ * deserialise steps run back-to-back. Bounding the queue here keeps
35
+ * peak Page accumulation predictable on giant trees without measurably
36
+ * changing throughput on typical Mosaic sites (~hundreds of files),
37
+ * where the bound is never reached.
38
+ *
39
+ * Insurance, not a fix for any observed Mosaic-site regression — keep
40
+ * it if you might consume monorepo-scale docs trees, drop it if the
41
+ * extra ~20 lines aren't worth the future-proofing.
42
+ */
43
+ async function mapWithConcurrency(items, limit, worker) {
44
+ const results = new Array(items.length);
45
+ let cursor = 0;
46
+ const runners = new Array(Math.min(limit, items.length)).fill(0).map(async () => {
47
+ for (;;) {
48
+ const i = cursor++;
49
+ if (i >= items.length)
50
+ return;
51
+ results[i] = await worker(items[i], i);
52
+ }
53
+ });
54
+ await Promise.all(runners);
55
+ return results;
19
56
  }
57
+ const READ_CONCURRENCY = 16;
58
+ const STAT_CONCURRENCY = 64;
20
59
  export const schema = z.object({
21
60
  /**
22
61
  * Collection of file extensions to look for
23
62
  */
24
- extensions: z
25
- .string({ required_error: 'Please provide the collection of file extensions to look for' })
26
- .array()
27
- .nonempty(),
63
+ extensions: z.tuple([z.string()], z.string()),
28
64
  /**
29
65
  * Add to use a folder prefix
30
66
  */
@@ -32,23 +68,119 @@ export const schema = z.object({
32
68
  /**
33
69
  * The root directory containing docs
34
70
  */
35
- rootDir: z.string({ required_error: 'Please provide a root directory name' })
71
+ rootDir: z.string({
72
+ error: issue => (issue.input === undefined ? 'Please provide a root directory name' : undefined)
73
+ })
36
74
  });
37
75
  const LocalFolderSource = {
76
+ /**
77
+ * WARNING for upstream consumers: the per-file deserialise cache
78
+ * (`cache` below) is scoped to a single `create()` invocation. If you
79
+ * compose this source under an operator that re-subscribes the
80
+ * returned observable on every tick of a polling stream (e.g.
81
+ * `switchMap(() => watchFolder$)`, `exhaustMap`, …), the cache is
82
+ * thrown away and rebuilt from scratch on every re-subscription,
83
+ * producing the worst possible memory churn and GC pressure on large
84
+ * docs trees. Subscribe once for the lifetime of the source and
85
+ * merge in side-streams as *signals*, not triggers.
86
+ */
38
87
  create(options, { serialiser }) {
39
88
  validateMosaicSchema(schema, options);
40
- return merge(of(null), fromFsWatch(options.rootDir, { recursive: true })).pipe(delay(1000), switchMap(() => glob(createFileGlob('**', options.extensions), {
89
+ // Per-`create()` cache, keyed by the relative filepath returned
90
+ // from `glob`. Lives for the lifetime of the Observable's
91
+ // subscription; each `LocalFolderSource` instance gets its own,
92
+ // so it composes cleanly with `prefixDir` / multi-source setups.
93
+ const cache = new Map();
94
+ // Files whose real location is outside `rootDir` (via a symlinked file
95
+ // or folder). Warn once each rather than on every re-scan.
96
+ const reportedOutsideRoot = new Set();
97
+ return merge(of(null), fromFsWatch(options.rootDir, { recursive: true })).pipe(
98
+ // Coalesce the burst of `rename`/`change` events most editors emit
99
+ // for a single save (atomic write → rename + chmod). 50 ms is below
100
+ // the human-perception threshold while comfortably wider than any
101
+ // editor's write burst.
102
+ debounceTime(50), switchMap(() => glob(createFileGlob('**', options.extensions), {
41
103
  cwd: options.rootDir,
42
- onlyFiles: true
43
- })), concatMap(async (filepaths) =>
44
- // eslint-disable-next-line @typescript-eslint/return-await
45
- await Promise.all(filepaths.map(async (filepath) => {
46
- const fullPath = path.posix.join(options.rootDir, filepath);
47
- return lodashMerge({}, await serialiser.deserialise(fullPath, await fs.promises.readFile(fullPath)), {
48
- lastModified: new Date(await getLastModifiedDate(fullPath)).getTime(),
49
- fullPath: `/${options.prefixDir ? path.posix.join(options.prefixDir, filepath) : filepath}`.replace(/^\/{2,}/, '/')
104
+ onlyFiles: true,
105
+ expandDirectories: false
106
+ })), concatMap(async (filepaths) => {
107
+ // Symlinks are followed by the glob, so a link committed to the
108
+ // content folder could point at any file the process can read
109
+ // (e.g. `leak.mdx -> /proc/self/environ`). Only publish files
110
+ // whose real path is inside the real root.
111
+ const realRoot = await fs.promises.realpath(options.rootDir);
112
+ // Stage 1: stat every file. `stat` is metadata-only (no read)
113
+ // and libuv caps the actual concurrent stats at its thread-pool
114
+ // size (4 by default), so the `STAT_CONCURRENCY` ceiling is
115
+ // mostly there to keep the in-memory queue of pending closures
116
+ // bounded on giant trees rather than to control I/O parallelism.
117
+ // Files that vanished between `glob` and `stat` are tolerated
118
+ // by treating the entry as "skip"; the next emission picks up
119
+ // the steady state.
120
+ const stats = await mapWithConcurrency(filepaths, STAT_CONCURRENCY, async (filepath) => {
121
+ const fullPath = path.posix.join(options.rootDir, filepath);
122
+ try {
123
+ const realPath = await fs.promises.realpath(fullPath);
124
+ const relativeToRoot = path.relative(realRoot, realPath);
125
+ if (relativeToRoot === '..' ||
126
+ relativeToRoot.startsWith(`..${path.sep}`) ||
127
+ path.isAbsolute(relativeToRoot)) {
128
+ if (!reportedOutsideRoot.has(filepath)) {
129
+ reportedOutsideRoot.add(filepath);
130
+ console.warn(`[Mosaic][Source-Local-Folder] Skipping ${fullPath}: it resolves outside ${options.rootDir}.`);
131
+ }
132
+ return undefined;
133
+ }
134
+ const s = await fs.promises.stat(realPath);
135
+ return { filepath, fullPath, realPath, mtimeMs: s.mtimeMs, size: s.size };
136
+ }
137
+ catch {
138
+ return undefined;
139
+ }
140
+ });
141
+ // Stage 2: reuse the cached `Page` when `(mtimeMs, size)` are
142
+ // unchanged, otherwise do the full read + deserialise. The
143
+ // expensive work is skipped for every unchanged file —
144
+ // typically every file but the one the user just saved.
145
+ //
146
+ // `structuredClone` rather than `lodash.merge({}, …)`: see the
147
+ // `CacheEntry` comment for the rationale.
148
+ const pages = await mapWithConcurrency(stats, READ_CONCURRENCY, async (entry) => {
149
+ if (!entry)
150
+ return undefined;
151
+ const { filepath, fullPath, realPath, mtimeMs, size } = entry;
152
+ const cached = cache.get(filepath);
153
+ if (cached && cached.mtimeMs === mtimeMs && cached.size === size) {
154
+ return structuredClone(cached.page);
155
+ }
156
+ const data = await fs.promises.readFile(realPath);
157
+ const deserialised = (await serialiser.deserialise(fullPath, data));
158
+ // The cache holds the canonical (pre-clone) page; consumers
159
+ // always receive a `structuredClone` of it on every emission.
160
+ //
161
+ // `lastModified` is typed as `Date` upstream but we assign a
162
+ // number (`mtimeMs`) — `lodash.merge` accepted this silently
163
+ // and the rest of the pipeline downstream depends on the
164
+ // number shape, so we preserve it via an explicit cast
165
+ // rather than quietly fixing the type and breaking consumers.
166
+ const page = {
167
+ ...deserialised,
168
+ lastModified: mtimeMs,
169
+ fullPath: buildFullPath(filepath, options.prefixDir)
170
+ };
171
+ cache.set(filepath, { mtimeMs, size, page });
172
+ return structuredClone(page);
50
173
  });
51
- }))));
174
+ // Stage 3: prune entries for files that disappeared between
175
+ // emissions, so the cache doesn't grow unboundedly across a
176
+ // long dev session.
177
+ const currentSet = new Set(filepaths);
178
+ for (const key of cache.keys()) {
179
+ if (!currentSet.has(key))
180
+ cache.delete(key);
181
+ }
182
+ return pages.filter((p) => p !== undefined);
183
+ }));
52
184
  }
53
185
  };
54
186
  export default LocalFolderSource;
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAc,EAAE,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAErE,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C,KAAK,UAAU,mBAAmB,CAAC,QAAgB;IACjD,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACpD,OAAO,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,cAAwB;IAC3D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,GAAG,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,GAAG,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B;;OAEG;IACH,UAAU,EAAE,CAAC;SACV,MAAM,CAAC,EAAE,cAAc,EAAE,8DAA8D,EAAE,CAAC;SAC1F,KAAK,EAAE;SACP,QAAQ,EAAE;IACb;;OAEG;IACH,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,sCAAsC,EAAE,CAAC;CAC9E,CAAC,CAAC;AAIH,MAAM,iBAAiB,GAAqC;IAC1D,MAAM,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE;QAC5B,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,OAAO,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAC5E,KAAK,CAAC,IAAI,CAAC,EACX,SAAS,CAAC,GAAG,EAAE,CACb,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE;YAC7C,GAAG,EAAE,OAAO,CAAC,OAAO;YACpB,SAAS,EAAE,IAAI;SAChB,CAAC,CACH,EACD,SAAS,CACP,KAAK,EAAE,SAAmB,EAAE,EAAE;QAC5B,2DAA2D;QAC3D,MAAM,OAAO,CAAC,GAAG,CACf,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;YACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC5D,OAAO,WAAW,CAChB,EAAE,EACF,MAAM,UAAU,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAC5E;gBACE,YAAY,EAAE,IAAI,IAAI,CAAC,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,EAAE;gBACrE,QAAQ,EAAE,IACR,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QACrE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aAC3B,CACF,CAAC;QACJ,CAAC,CAAC,CACH,CACJ,CACF,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,eAAe,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAc,EAAE,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjF,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAErE,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C,SAAS,cAAc,CAAC,GAAW,EAAE,cAAwB;IAC3D,2FAA2F;IAC3F,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC1D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,GAAG,IAAI,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAChD,CAAC;AA8BD;;;;GAIG;AACH,SAAS,aAAa,CAAC,QAAgB,EAAE,SAA6B;IACpE,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC3E,OAAO,IAAI,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,kBAAkB,CAC/B,KAAmB,EACnB,KAAa,EACb,MAA8C;IAE9C,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;QAC9E,SAAS,CAAC;YACR,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;YACnB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAC5B,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B;;OAEG;IACH,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IAC7C;;OAEG;IACH,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC,CAAC,SAAS,CAAC;KACjG,CAAC;CACH,CAAC,CAAC;AAIH,MAAM,iBAAiB,GAAqC;IAC1D;;;;;;;;;;OAUG;IACH,MAAM,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE;QAC5B,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAEtC,gEAAgE;QAChE,0DAA0D;QAC1D,gEAAgE;QAChE,iEAAiE;QACjE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;QAC5C,uEAAuE;QACvE,2DAA2D;QAC3D,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE9C,OAAO,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;QAC5E,mEAAmE;QACnE,oEAAoE;QACpE,kEAAkE;QAClE,wBAAwB;QACxB,YAAY,CAAC,EAAE,CAAC,EAChB,SAAS,CAAC,GAAG,EAAE,CACb,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE;YAC7C,GAAG,EAAE,OAAO,CAAC,OAAO;YACpB,SAAS,EAAE,IAAI;YACf,iBAAiB,EAAE,KAAK;SACzB,CAAC,CACH,EACD,SAAS,CAAC,KAAK,EAAE,SAAmB,EAAE,EAAE;YACtC,gEAAgE;YAChE,8DAA8D;YAC9D,8DAA8D;YAC9D,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAE7D,8DAA8D;YAC9D,gEAAgE;YAChE,4DAA4D;YAC5D,+DAA+D;YAC/D,iEAAiE;YACjE,8DAA8D;YAC9D,8DAA8D;YAC9D,oBAAoB;YACpB,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,SAAS,EAAE,gBAAgB,EAAE,KAAK,EAAC,QAAQ,EAAC,EAAE;gBACnF,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC5D,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBACtD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBACzD,IACE,cAAc,KAAK,IAAI;wBACvB,cAAc,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC1C,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,EAC/B,CAAC;wBACD,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;4BACvC,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;4BAClC,OAAO,CAAC,IAAI,CACV,0CAA0C,QAAQ,yBAAyB,OAAO,CAAC,OAAO,GAAG,CAC9F,CAAC;wBACJ,CAAC;wBACD,OAAO,SAAS,CAAC;oBACnB,CAAC;oBACD,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC3C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5E,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,SAAS,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,2DAA2D;YAC3D,uDAAuD;YACvD,wDAAwD;YACxD,EAAE;YACF,+DAA+D;YAC/D,0CAA0C;YAC1C,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;gBAC5E,IAAI,CAAC,KAAK;oBAAE,OAAO,SAAS,CAAC;gBAC7B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;gBAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACnC,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBACjE,OAAO,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACtC,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAClD,MAAM,YAAY,GAAG,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAS,CAAC;gBAC5E,4DAA4D;gBAC5D,8DAA8D;gBAC9D,EAAE;gBACF,6DAA6D;gBAC7D,6DAA6D;gBAC7D,yDAAyD;gBACzD,uDAAuD;gBACvD,8DAA8D;gBAC9D,MAAM,IAAI,GAAG;oBACX,GAAG,YAAY;oBACf,YAAY,EAAE,OAAO;oBACrB,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC;iBAClC,CAAC;gBACrB,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7C,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,4DAA4D;YAC5D,4DAA4D;YAC5D,oBAAoB;YACpB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YACtC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9C,CAAC;YAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAa,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QACzD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,eAAe,iBAAiB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jpmorganchase/mosaic-source-local-folder",
3
- "version": "0.1.0-beta.98",
3
+ "version": "1.0.0-beta.99",
4
4
  "author": "",
5
5
  "description": "Mosaic local folder source for core file system",
6
6
  "license": "Apache-2.0",
@@ -16,7 +16,7 @@
16
16
  }
17
17
  },
18
18
  "engines": {
19
- "node": ">=18.0.0 || >=20.0.0"
19
+ "node": ">=24"
20
20
  },
21
21
  "files": [
22
22
  "dist"
@@ -24,20 +24,18 @@
24
24
  "scripts": {
25
25
  "build": "tsc",
26
26
  "clean": "rm -fr dist",
27
- "lint": "eslint --ignore-pattern \"**/__tests__/**\""
27
+ "lint": "eslint ."
28
28
  },
29
29
  "repository": {
30
30
  "type": "git",
31
- "url": "git@github.com:jpmorganchase/mosaic.git",
31
+ "url": "git+https://github.com/jpmorganchase/mosaic.git",
32
32
  "directory": "packages/source-local-folder"
33
33
  },
34
34
  "dependencies": {
35
- "@jpmorganchase/mosaic-schemas": "^0.1.0-beta.98",
36
- "@jpmorganchase/mosaic-types": "^0.1.0-beta.98",
37
- "chokidar": "^3.5.3",
38
- "fast-glob": "^3.2.12",
39
- "lodash-es": "^4.17.23",
35
+ "@jpmorganchase/mosaic-schemas": "^1.0.0-beta.99",
36
+ "@jpmorganchase/mosaic-types": "^1.0.0-beta.99",
40
37
  "rxjs": "^7.5.5",
41
- "zod": "^3.22.3"
38
+ "tinyglobby": "^0.2.17",
39
+ "zod": "^4.6.7"
42
40
  }
43
- }
41
+ }