@lowdefy/server-dev 7.0.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/docs/dataSets/checkSharedCollectionIds.js +20 -9
- package/lib/docs/dataSets/dataSetJourneys.chromium.mongodb.test.mjs +53 -5
- package/lib/docs/dataSets/describeTargetKey.js +29 -0
- package/lib/docs/dataSets/groupDataSetByCollection.js +19 -11
- package/lib/docs/dataSets/openDataSession.js +32 -18
- package/lib/docs/dataSets/openDataSession.mongodb.test.mjs +150 -0
- package/lib/docs/dataSets/readDataSet.js +34 -19
- package/lib/docs/dataSets/readDataSet.test.mjs +56 -25
- package/lib/docs/dataSets/sessionDatabaseName.js +32 -0
- package/lib/docs/dataSets/sweepOrphanDatabases.js +7 -4
- package/lib/docs/dataSets/test/app/lowdefy.yaml +22 -0
- package/lib/docs/dataSets/test/app/tests/data/shop.yaml +4 -0
- package/lib/docs/devToolDefinitions.js +3 -3
- package/lib/docs/getBrowser.js +1 -1
- package/lib/docs/getBuildStatus.js +29 -3
- package/lib/docs/getBuildStatus.test.mjs +99 -3
- package/lib/docs/getOverview.js +2 -2
- package/lib/docs/rails/findExternalBlocks.js +3 -3
- package/lib/docs/runJourneyDataSet.test.mjs +1 -1
- package/lib/docs/withDataSession.js +2 -2
- package/lib/server/applyDataSetRedirect.js +14 -5
- package/lib/server/applyDataSetRedirect.test.mjs +119 -13
- package/lib/server/applyDataSetRedirectTenantReach.test.mjs +4 -1
- package/lib/server/applyDataSetRedirectWalled.test.mjs +4 -1
- package/lib/server/createLowdefyContext.js +3 -3
- package/lib/server/createLowdefyContext.test.mjs +6 -3
- package/lib/server/mutantDataSetWrappers.test.mjs +4 -1
- package/manager/getContext.mjs +2 -2
- package/manager/processes/initialBuild.mjs +3 -1
- package/manager/processes/initialBuild.test.mjs +3 -0
- package/manager/processes/lowdefyBuild.mjs +4 -3
- package/manager/processes/lowdefyBuild.skipPluginChecks.test.mjs +10 -0
- package/package.json +37 -37
- package/package.original.json +37 -37
- package/lib/docs/dataSets/findCollectionCollisions.js +0 -52
|
@@ -16,12 +16,21 @@
|
|
|
16
16
|
|
|
17
17
|
import { type } from '@lowdefy/helpers';
|
|
18
18
|
|
|
19
|
+
// Where a connection loads: its collection in its database target. Two connections share a
|
|
20
|
+
// collection only when both match.
|
|
21
|
+
function describePlace({ dataSet, connectionId }) {
|
|
22
|
+
return JSON.stringify([
|
|
23
|
+
dataSet.targets?.[connectionId] ?? null,
|
|
24
|
+
dataSet.collections[connectionId],
|
|
25
|
+
]);
|
|
26
|
+
}
|
|
27
|
+
|
|
19
28
|
// `label` names where a document comes from in a message: "fixture " or "generate.".
|
|
20
|
-
function addIds({ held, connectionId,
|
|
21
|
-
held[
|
|
29
|
+
function addIds({ held, connectionId, place, documents, label }) {
|
|
30
|
+
held[place] = held[place] ?? new Map();
|
|
22
31
|
documents.forEach((document, index) => {
|
|
23
32
|
if (type.isUndefined(document._id)) return;
|
|
24
|
-
held[
|
|
33
|
+
held[place].set(JSON.stringify(document._id), {
|
|
25
34
|
connectionId,
|
|
26
35
|
where: `${label}${connectionId}[${index}]`,
|
|
27
36
|
});
|
|
@@ -29,23 +38,25 @@ function addIds({ held, connectionId, collection, documents, label }) {
|
|
|
29
38
|
}
|
|
30
39
|
|
|
31
40
|
// parseDataSet refuses a generated _id that is also a fixture's or generated twice, within one
|
|
32
|
-
// connection. Two connections can name one collection, and the store loads
|
|
33
|
-
// where a fixture would silently replace a generated document with its _id.
|
|
34
|
-
// collisions across connections, which only the dev build's collection
|
|
41
|
+
// connection. Two connections can name one collection in one database target, and the store loads
|
|
42
|
+
// them into it together, where a fixture would silently replace a generated document with its _id.
|
|
43
|
+
// This refuses the same collisions across connections, which only the dev build's collection and
|
|
44
|
+
// target maps can see.
|
|
35
45
|
function checkSharedCollectionIds({ dataSetName, dataSet }) {
|
|
36
46
|
const held = {};
|
|
37
47
|
Object.entries(dataSet.fixtures).forEach(([connectionId, documents]) => {
|
|
38
48
|
addIds({
|
|
39
49
|
held,
|
|
40
50
|
connectionId,
|
|
41
|
-
|
|
51
|
+
place: describePlace({ dataSet, connectionId }),
|
|
42
52
|
documents,
|
|
43
53
|
label: 'fixture ',
|
|
44
54
|
});
|
|
45
55
|
});
|
|
46
56
|
Object.entries(dataSet.generated).forEach(([connectionId, documents]) => {
|
|
47
57
|
const collection = dataSet.collections[connectionId];
|
|
48
|
-
const
|
|
58
|
+
const place = describePlace({ dataSet, connectionId });
|
|
59
|
+
const ids = held[place] ?? new Map();
|
|
49
60
|
documents.forEach((document, index) => {
|
|
50
61
|
const key = JSON.stringify(document._id);
|
|
51
62
|
const holder = ids.get(key);
|
|
@@ -57,7 +68,7 @@ function checkSharedCollectionIds({ dataSetName, dataSet }) {
|
|
|
57
68
|
addIds({
|
|
58
69
|
held,
|
|
59
70
|
connectionId,
|
|
60
|
-
|
|
71
|
+
place,
|
|
61
72
|
documents,
|
|
62
73
|
label: 'generate.',
|
|
63
74
|
});
|
|
@@ -49,8 +49,13 @@ const standInDatabase = 'data_set_journeys';
|
|
|
49
49
|
const standInUrl = new URL(process.env.MONGO_URL);
|
|
50
50
|
standInUrl.pathname = `/${standInDatabase}`;
|
|
51
51
|
const standInUri = standInUrl.toString();
|
|
52
|
+
// The CRM_MONGODB_URI secret's stand-in: another database with a tickets collection of its own.
|
|
53
|
+
const crmStandInDatabase = 'data_set_journeys_crm';
|
|
54
|
+
const crmStandInUrl = new URL(process.env.MONGO_URL);
|
|
55
|
+
crmStandInUrl.pathname = `/${crmStandInDatabase}`;
|
|
52
56
|
|
|
53
57
|
process.env.LOWDEFY_SECRET_MONGODB_URI = standInUri;
|
|
58
|
+
process.env.LOWDEFY_SECRET_CRM_MONGODB_URI = crmStandInUrl.toString();
|
|
54
59
|
process.env.LOWDEFY_DIRECTORY_CONFIG = appDirectory;
|
|
55
60
|
process.env.LOWDEFY_LOG_LEVEL = 'error';
|
|
56
61
|
process.env.CRON_SECRET = 'data-set-journeys-cron-secret';
|
|
@@ -144,6 +149,7 @@ function pageHtml() {
|
|
|
144
149
|
<div id="bl-create"><button onclick="callEndpoint('create_ticket', { id: 'new-1', title: 'Made' }, 'created')">Create</button></div>
|
|
145
150
|
<div id="bl-create_later"><button onclick="callEndpoint('create_ticket_later', { id: 'later-1' }, 'dispatched')">Later</button></div>
|
|
146
151
|
<div id="bl-list"><button onclick="callEndpoint('list_tickets', {}, 'listed')">List</button></div>
|
|
152
|
+
<div id="bl-list_crm"><button onclick="callEndpoint('list_crm_tickets', {}, 'listedCrm')">List CRM</button></div>
|
|
147
153
|
<div id="bl-watch"><button onclick="watch()">Watch</button></div>
|
|
148
154
|
<div id="bl-archive"><button onclick="callEndpoint('archive/archive_ticket', { id: 'arch-1' }, 'archived')">Archive</button></div>
|
|
149
155
|
<div id="bl-create_later_auth"><button onclick="callEndpoint('create_ticket_later_auth', {}, 'dispatchedAuth')">Later with auth</button></div>
|
|
@@ -301,6 +307,11 @@ beforeAll(async () => {
|
|
|
301
307
|
.db(standInDatabase)
|
|
302
308
|
.collection('tickets')
|
|
303
309
|
.insertOne({ _id: 'real-1', organization_id: 'org_a', title: 'The developer’s own' });
|
|
310
|
+
await standIn.db(crmStandInDatabase).dropDatabase();
|
|
311
|
+
await standIn
|
|
312
|
+
.db(crmStandInDatabase)
|
|
313
|
+
.collection('tickets')
|
|
314
|
+
.insertOne({ _id: 'crm-real-1', organization_id: 'org_a' });
|
|
304
315
|
|
|
305
316
|
store = await getDataStore();
|
|
306
317
|
// Every write on the data store, in order, so a test can tell which session database a detached
|
|
@@ -346,6 +357,7 @@ afterAll(async () => {
|
|
|
346
357
|
await closeClients();
|
|
347
358
|
await store?.stop();
|
|
348
359
|
await standIn?.db(standInDatabase).dropDatabase();
|
|
360
|
+
await standIn?.db(crmStandInDatabase).dropDatabase();
|
|
349
361
|
await standIn?.close();
|
|
350
362
|
process.chdir(originalCwd);
|
|
351
363
|
fs.rmSync(serverDirectory, { recursive: true, force: true });
|
|
@@ -396,7 +408,7 @@ chromiumTest(
|
|
|
396
408
|
});
|
|
397
409
|
expect(first.status).toBe(200);
|
|
398
410
|
expect(first.result.passed).toBe(true);
|
|
399
|
-
expect(first.result.data).toEqual({ name: 'shop', loadMs: expect.any(Number), documents:
|
|
411
|
+
expect(first.result.data).toEqual({ name: 'shop', loadMs: expect.any(Number), documents: 5 });
|
|
400
412
|
|
|
401
413
|
const second = await runJourney({
|
|
402
414
|
data: 'shop',
|
|
@@ -466,7 +478,7 @@ chromiumTest(
|
|
|
466
478
|
(event) => event.operationType === 'insert' && event.id === 'later-1'
|
|
467
479
|
);
|
|
468
480
|
expect(insert).toBeDefined();
|
|
469
|
-
expect(insert.db).toMatch(/^ld_[0-9a-f]{12}
|
|
481
|
+
expect(insert.db).toMatch(/^ld_[0-9a-f]{12}_\d+$/);
|
|
470
482
|
const insertIndex = storeEvents.indexOf(insert);
|
|
471
483
|
const dropIndex = storeEvents.findIndex(
|
|
472
484
|
(event) => event.operationType === 'dropDatabase' && event.db === insert.db
|
|
@@ -523,7 +535,7 @@ chromiumTest(
|
|
|
523
535
|
(event) => event.operationType === 'insert' && event.id === 'arch-1'
|
|
524
536
|
);
|
|
525
537
|
expect(insert).toBeDefined();
|
|
526
|
-
expect(insert.db).toMatch(/^ld_[0-9a-f]{12}
|
|
538
|
+
expect(insert.db).toMatch(/^ld_[0-9a-f]{12}_\d+$/);
|
|
527
539
|
expect(insert.coll).toEqual('archived');
|
|
528
540
|
}
|
|
529
541
|
);
|
|
@@ -568,7 +580,7 @@ chromiumTest(
|
|
|
568
580
|
(event) => event.operationType === 'insert' && event.id === 'later-auth-before'
|
|
569
581
|
);
|
|
570
582
|
expect(before).toBeDefined();
|
|
571
|
-
expect(before.db).toMatch(/^ld_[0-9a-f]{12}
|
|
583
|
+
expect(before.db).toMatch(/^ld_[0-9a-f]{12}_\d+$/);
|
|
572
584
|
expect(storeEvents.some((event) => event.id === 'later-auth-after')).toBe(false);
|
|
573
585
|
}
|
|
574
586
|
);
|
|
@@ -636,7 +648,7 @@ chromiumTest(
|
|
|
636
648
|
(event) => event.operationType === 'insert' && event.id === 'later-logged'
|
|
637
649
|
);
|
|
638
650
|
expect(insert).toBeDefined();
|
|
639
|
-
expect(insert.db).toMatch(/^ld_[0-9a-f]{12}
|
|
651
|
+
expect(insert.db).toMatch(/^ld_[0-9a-f]{12}_\d+$/);
|
|
640
652
|
expect(storeEvents.some((event) => event.id === 'log-detached')).toBe(false);
|
|
641
653
|
expect(await standInIds()).toEqual(['real-1']);
|
|
642
654
|
}
|
|
@@ -673,7 +685,43 @@ chromiumTest(
|
|
|
673
685
|
|
|
674
686
|
// Last, after every journey above: the stand-in, auth collections included, is exactly as the
|
|
675
687
|
// snapshot found it, and no session database ever landed on it.
|
|
688
|
+
chromiumTest(
|
|
689
|
+
'two connections that name tickets in different databases each read their own fixtures',
|
|
690
|
+
async () => {
|
|
691
|
+
storeEvents.length = 0;
|
|
692
|
+
const { result } = await runJourney({
|
|
693
|
+
data: 'shop',
|
|
694
|
+
user: 'owner',
|
|
695
|
+
steps: [
|
|
696
|
+
{ click: 'list' },
|
|
697
|
+
{ expect: { state: { path: 'listed.ids', equals: ['t-a1', 't-a2'] } } },
|
|
698
|
+
{ click: 'list_crm' },
|
|
699
|
+
{ expect: { state: { path: 'listedCrm.ids', equals: ['crm-a2', 't-a1'] } } },
|
|
700
|
+
{ click: 'create' },
|
|
701
|
+
{ expect: { state: { path: 'created.success', equals: true } } },
|
|
702
|
+
{ click: 'list_crm' },
|
|
703
|
+
{ expect: { state: { path: 'listedCrm.ids', equals: ['crm-a2', 't-a1'] } } },
|
|
704
|
+
],
|
|
705
|
+
});
|
|
706
|
+
expect(result.passed).toBe(true);
|
|
707
|
+
// Each target's fixtures loaded into a database of its own, and both dropped with the session.
|
|
708
|
+
const fixtureDatabases = new Set(
|
|
709
|
+
storeEvents
|
|
710
|
+
.filter((event) => event.operationType === 'insert' && event.id === 't-a1')
|
|
711
|
+
.map((event) => event.db)
|
|
712
|
+
);
|
|
713
|
+
expect(fixtureDatabases.size).toBe(2);
|
|
714
|
+
fixtureDatabases.forEach((name) => {
|
|
715
|
+
expect(
|
|
716
|
+
storeEvents.some((event) => event.operationType === 'dropDatabase' && event.db === name)
|
|
717
|
+
).toBe(true);
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
);
|
|
721
|
+
|
|
676
722
|
chromiumTest('the data set journeys left the stand-in database unchanged', async () => {
|
|
677
723
|
expect(await snapshotStandIn()).toEqual(standInBefore);
|
|
724
|
+
const crmIds = await standIn.db(crmStandInDatabase).collection('tickets').distinct('_id');
|
|
725
|
+
expect(crmIds).toEqual(['crm-real-1']);
|
|
678
726
|
expect(await standInSessionDatabases()).toEqual([]);
|
|
679
727
|
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Copyright 2020-2026 Lowdefy, Inc
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
you may not use this file except in compliance with the License.
|
|
6
|
+
You may obtain a copy of the License at
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
See the License for the specific language governing permissions and
|
|
14
|
+
limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { type } from '@lowdefy/helpers';
|
|
18
|
+
|
|
19
|
+
import describeDatabaseTarget from './describeDatabaseTarget.js';
|
|
20
|
+
|
|
21
|
+
// The key a data session gives each database target its own database by, or null when the target
|
|
22
|
+
// is computed per request.
|
|
23
|
+
function describeTargetKey({ properties }) {
|
|
24
|
+
const target = describeDatabaseTarget({ properties });
|
|
25
|
+
if (type.isNone(target)) return null;
|
|
26
|
+
return JSON.stringify([target.uri, target.databaseName]);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export default describeTargetKey;
|
|
@@ -14,31 +14,39 @@
|
|
|
14
14
|
limitations under the License.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
// One group per database target and collection: `target` is the connection's target key (null when
|
|
18
|
+
// computed per request), the database the session loads the group into.
|
|
19
|
+
function ensureGroup({ groups, dataSet, connectionId }) {
|
|
20
|
+
const target = dataSet.targets?.[connectionId] ?? null;
|
|
21
|
+
const collection = dataSet.collections[connectionId];
|
|
22
|
+
const key = JSON.stringify([target, collection]);
|
|
23
|
+
if (!groups.has(key)) {
|
|
24
|
+
groups.set(key, { target, collection, fixtures: [], generated: [], indexes: [] });
|
|
25
|
+
}
|
|
26
|
+
return groups.get(key);
|
|
20
27
|
}
|
|
21
28
|
|
|
22
|
-
// The data set keyed by collection, the unit the store loads: fixtures,
|
|
23
|
-
// indexes are keyed by connection id, and two connections that name one
|
|
24
|
-
// together.
|
|
29
|
+
// The data set keyed by database target and collection, the unit the store loads: fixtures,
|
|
30
|
+
// generated documents and indexes are keyed by connection id, and two connections that name one
|
|
31
|
+
// collection in one target load into it together. Connections in different targets load into
|
|
32
|
+
// different databases, so they never merge.
|
|
25
33
|
function groupDataSetByCollection({ dataSet }) {
|
|
26
|
-
const groups =
|
|
34
|
+
const groups = new Map();
|
|
27
35
|
Object.entries(dataSet.fixtures).forEach(([connectionId, documents]) => {
|
|
28
|
-
const group = ensureGroup({ groups,
|
|
36
|
+
const group = ensureGroup({ groups, dataSet, connectionId });
|
|
29
37
|
documents.forEach((document, index) => {
|
|
30
38
|
group.fixtures.push({ connectionId, index, document });
|
|
31
39
|
});
|
|
32
40
|
});
|
|
33
41
|
Object.entries(dataSet.generated).forEach(([connectionId, documents]) => {
|
|
34
|
-
const group = ensureGroup({ groups,
|
|
42
|
+
const group = ensureGroup({ groups, dataSet, connectionId });
|
|
35
43
|
group.generated.push({ connectionId, documents });
|
|
36
44
|
});
|
|
37
45
|
Object.entries(dataSet.indexes).forEach(([connectionId, indexes]) => {
|
|
38
|
-
const group = ensureGroup({ groups,
|
|
46
|
+
const group = ensureGroup({ groups, dataSet, connectionId });
|
|
39
47
|
group.indexes.push(...indexes);
|
|
40
48
|
});
|
|
41
|
-
return groups;
|
|
49
|
+
return [...groups.values()];
|
|
42
50
|
}
|
|
43
51
|
|
|
44
52
|
export default groupDataSetByCollection;
|
|
@@ -23,10 +23,20 @@ import getDataStore from './getDataStore.js';
|
|
|
23
23
|
import groupDataSetByCollection from './groupDataSetByCollection.js';
|
|
24
24
|
import isDataStoreGone from './isDataStoreGone.js';
|
|
25
25
|
import loadDataSetCollection from './loadDataSetCollection.js';
|
|
26
|
+
import sessionDatabaseName from './sessionDatabaseName.js';
|
|
26
27
|
import sweepOrphanDatabases from './sweepOrphanDatabases.js';
|
|
27
28
|
|
|
28
29
|
const DRAIN_TIMEOUT_MS = 30000;
|
|
29
30
|
|
|
31
|
+
// Every database the session holds, the first included.
|
|
32
|
+
function listSessionDatabases(session) {
|
|
33
|
+
return [...new Set([session.databaseName, ...session.databases.values()])];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function dropSessionDatabases({ client, session }) {
|
|
37
|
+
await Promise.all(listSessionDatabases(session).map((name) => client.db(name).dropDatabase()));
|
|
38
|
+
}
|
|
39
|
+
|
|
30
40
|
function createClose({ session, client }) {
|
|
31
41
|
let closing;
|
|
32
42
|
async function close() {
|
|
@@ -37,14 +47,14 @@ function createClose({ session, client }) {
|
|
|
37
47
|
console.warn(
|
|
38
48
|
`Data set "${session.name}": background work was still running ${
|
|
39
49
|
DRAIN_TIMEOUT_MS / 1000
|
|
40
|
-
} s after its journey ended. Dropping its
|
|
50
|
+
} s after its journey ended. Dropping its databases anyway; that work now fails.`
|
|
41
51
|
);
|
|
42
52
|
}
|
|
43
53
|
// Closed before the drop: a connection read under this session throws from here on, so
|
|
44
|
-
// nothing that outlived the drain reaches any database.
|
|
54
|
+
// nothing that outlived the drain reaches any database, and no target is added after the drop.
|
|
45
55
|
session.state = 'closed';
|
|
46
56
|
dataSessionRegistry.delete(session.id);
|
|
47
|
-
await client
|
|
57
|
+
await dropSessionDatabases({ client, session });
|
|
48
58
|
}
|
|
49
59
|
return function closeOnce() {
|
|
50
60
|
closing = closing ?? close();
|
|
@@ -52,20 +62,28 @@ function createClose({ session, client }) {
|
|
|
52
62
|
};
|
|
53
63
|
}
|
|
54
64
|
|
|
55
|
-
// Opens a data session for one journey run (or a mutant run):
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
65
|
+
// Opens a data session for one journey run (or a mutant run): fresh databases on the dev server's
|
|
66
|
+
// memory store, one per database target the build names (dataSet.databaseTargets), loaded with the
|
|
67
|
+
// data set and registered under a random id. They are named <prefix>_<n> under the session's own
|
|
68
|
+
// prefix (ld_<hex>), so the orphan sweep and close() find them all; the first, <prefix>_0, is also
|
|
69
|
+
// the database a target computed per request reads. `cookie` is the id, the payload
|
|
70
|
+
// openPage({ dataCookie }) writes into each actor's lowdefy_journey_data cookie. close() waits for
|
|
71
|
+
// the session's background work (context.waitUntil), then drops the databases.
|
|
59
72
|
async function openDataSession({ dataSet }) {
|
|
60
73
|
const store = await getDataStore();
|
|
61
74
|
const { client, uri } = store;
|
|
62
75
|
const id = crypto.randomBytes(16).toString('hex');
|
|
76
|
+
const prefix = `ld_${crypto.randomBytes(6).toString('hex')}`;
|
|
77
|
+
const databaseTargets = dataSet.databaseTargets ?? [];
|
|
63
78
|
const session = {
|
|
64
79
|
id,
|
|
65
80
|
databaseUri: uri,
|
|
66
|
-
|
|
81
|
+
prefix,
|
|
82
|
+
databaseName: `${prefix}_0`,
|
|
83
|
+
databases: new Map(databaseTargets.map((target, index) => [target, `${prefix}_${index}`])),
|
|
84
|
+
databaseCount: Math.max(databaseTargets.length, 1),
|
|
67
85
|
name: dataSet.name,
|
|
68
|
-
// Registered while it loads, so a concurrent open's orphan sweep leaves its
|
|
86
|
+
// Registered while it loads, so a concurrent open's orphan sweep leaves its databases alone.
|
|
69
87
|
// readDataSession serves only open and closing sessions.
|
|
70
88
|
state: 'loading',
|
|
71
89
|
work: new Set(),
|
|
@@ -75,15 +93,14 @@ async function openDataSession({ dataSet }) {
|
|
|
75
93
|
dataSessionRegistry.set(id, session);
|
|
76
94
|
try {
|
|
77
95
|
await sweepOrphanDatabases({ client });
|
|
78
|
-
const db = client.db(session.databaseName);
|
|
79
96
|
const groups = groupDataSetByCollection({ dataSet });
|
|
80
|
-
// allSettled, so no collection is still loading when a failed load drops the
|
|
97
|
+
// allSettled, so no collection is still loading when a failed load drops the databases.
|
|
81
98
|
const results = await Promise.allSettled(
|
|
82
|
-
|
|
99
|
+
groups.map((group) =>
|
|
83
100
|
loadDataSetCollection({
|
|
84
|
-
db,
|
|
101
|
+
db: client.db(sessionDatabaseName({ session, target: group.target })),
|
|
85
102
|
dataSetName: dataSet.name,
|
|
86
|
-
collectionName,
|
|
103
|
+
collectionName: group.collection,
|
|
87
104
|
group,
|
|
88
105
|
})
|
|
89
106
|
)
|
|
@@ -105,10 +122,7 @@ async function openDataSession({ dataSet }) {
|
|
|
105
122
|
}
|
|
106
123
|
// Best effort: the load error is the one to report, and the next open's orphan sweep drops a
|
|
107
124
|
// database this could not.
|
|
108
|
-
await client
|
|
109
|
-
.db(session.databaseName)
|
|
110
|
-
.dropDatabase()
|
|
111
|
-
.catch(() => {});
|
|
125
|
+
await dropSessionDatabases({ client, session }).catch(() => {});
|
|
112
126
|
throw error;
|
|
113
127
|
}
|
|
114
128
|
session.state = 'open';
|
|
@@ -25,6 +25,8 @@ import dataSessionRegistry from './dataSessionRegistry.js';
|
|
|
25
25
|
import getDataStore from './getDataStore.js';
|
|
26
26
|
import openDataSession from './openDataSession.js';
|
|
27
27
|
import readDataSession from './readDataSession.js';
|
|
28
|
+
import readDataSet from './readDataSet.js';
|
|
29
|
+
import applyDataSetRedirect from '../../server/applyDataSetRedirect.js';
|
|
28
30
|
import { journeyActorToken } from '../../server/auth/journeyActor.js';
|
|
29
31
|
|
|
30
32
|
jest.setTimeout(120000);
|
|
@@ -281,6 +283,154 @@ test('two sessions on one data set are isolated, and close() drops the database'
|
|
|
281
283
|
expect(first.session.state).toEqual('closed');
|
|
282
284
|
});
|
|
283
285
|
|
|
286
|
+
describe('two databases that name one collection', () => {
|
|
287
|
+
let configDirectory;
|
|
288
|
+
let buildDirectory;
|
|
289
|
+
|
|
290
|
+
const artifacts = {
|
|
291
|
+
app_users: {
|
|
292
|
+
connectionId: 'app_users',
|
|
293
|
+
type: 'MongoDBCollection',
|
|
294
|
+
properties: { databaseUri: { _secret: 'MONGODB_URI' }, collection: 'users' },
|
|
295
|
+
},
|
|
296
|
+
auth_users: {
|
|
297
|
+
connectionId: 'auth_users',
|
|
298
|
+
type: 'MongoDBCollection',
|
|
299
|
+
properties: { databaseUri: { _secret: 'AUTH_MONGODB_URI' }, collection: 'users' },
|
|
300
|
+
},
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
beforeEach(() => {
|
|
304
|
+
configDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'lowdefy-data-targets-'));
|
|
305
|
+
buildDirectory = path.join(configDirectory, '.lowdefy', 'dev', 'build');
|
|
306
|
+
fs.mkdirSync(path.join(configDirectory, 'tests', 'data'), { recursive: true });
|
|
307
|
+
fs.mkdirSync(path.join(buildDirectory, 'connections'), { recursive: true });
|
|
308
|
+
Object.values(artifacts).forEach((artifact) => {
|
|
309
|
+
fs.writeFileSync(
|
|
310
|
+
path.join(buildDirectory, 'connections', `${artifact.connectionId}.json`),
|
|
311
|
+
JSON.stringify(artifact)
|
|
312
|
+
);
|
|
313
|
+
});
|
|
314
|
+
fs.writeFileSync(
|
|
315
|
+
path.join(configDirectory, 'tests', 'data', 'targets.yaml'),
|
|
316
|
+
`fixtures:
|
|
317
|
+
app_users: [{ _id: shared, name: App member }]
|
|
318
|
+
auth_users: [{ _id: shared, name: Auth user }, { _id: auth-only }]
|
|
319
|
+
indexes:
|
|
320
|
+
app_users: [{ key: { name: 1 }, name: name_1, unique: true }]
|
|
321
|
+
`
|
|
322
|
+
);
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
afterEach(() => {
|
|
326
|
+
fs.rmSync(configDirectory, { recursive: true, force: true });
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
// The database a request on the connection reads under the session, through the gate.
|
|
330
|
+
async function redirectedDb({ session, connectionId }) {
|
|
331
|
+
const context = {
|
|
332
|
+
readConfigFile: async (filePath) =>
|
|
333
|
+
artifacts[filePath.replace(/^connections\/(.+)\.json$/, '$1')] ?? null,
|
|
334
|
+
};
|
|
335
|
+
applyDataSetRedirect({
|
|
336
|
+
context,
|
|
337
|
+
session,
|
|
338
|
+
connections: { MongoDBCollection: { meta: { dataSet: 'redirect' } } },
|
|
339
|
+
});
|
|
340
|
+
const { properties } = await context.readConfigFile(`connections/${connectionId}.json`);
|
|
341
|
+
expect(properties.databaseUri).toEqual(session.databaseUri);
|
|
342
|
+
return client.db(properties.databaseName);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
test('each databaseUri secret loads into and reads its own session database, and close() drops both', async () => {
|
|
346
|
+
const dataSet = await readDataSet({ configDirectory, buildDirectory, name: 'targets' });
|
|
347
|
+
const { session, close } = await openDataSession({ dataSet });
|
|
348
|
+
let names;
|
|
349
|
+
try {
|
|
350
|
+
const appDb = await redirectedDb({ session, connectionId: 'app_users' });
|
|
351
|
+
const authDb = await redirectedDb({ session, connectionId: 'auth_users' });
|
|
352
|
+
expect(appDb.databaseName).not.toEqual(authDb.databaseName);
|
|
353
|
+
names = [appDb.databaseName, authDb.databaseName];
|
|
354
|
+
names.forEach((name) => expect(name.startsWith(`${session.prefix}_`)).toBe(true));
|
|
355
|
+
expect(await appDb.collection('users').find().toArray()).toEqual([
|
|
356
|
+
{ _id: 'shared', name: 'App member' },
|
|
357
|
+
]);
|
|
358
|
+
expect(await authDb.collection('users').find().sort({ _id: 1 }).toArray()).toEqual([
|
|
359
|
+
{ _id: 'auth-only' },
|
|
360
|
+
{ _id: 'shared', name: 'Auth user' },
|
|
361
|
+
]);
|
|
362
|
+
// Indexes load per target too.
|
|
363
|
+
const appIndexes = await appDb.collection('users').indexes();
|
|
364
|
+
const authIndexes = await authDb.collection('users').indexes();
|
|
365
|
+
expect(appIndexes.map((index) => index.name)).toContain('name_1');
|
|
366
|
+
expect(authIndexes.map((index) => index.name)).not.toContain('name_1');
|
|
367
|
+
expect(await listDatabaseNames()).toEqual(expect.arrayContaining(names));
|
|
368
|
+
} finally {
|
|
369
|
+
await close();
|
|
370
|
+
}
|
|
371
|
+
const left = await listDatabaseNames();
|
|
372
|
+
names.forEach((name) => expect(left).not.toContain(name));
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
test('a connection whose target is computed loads into the first target database with it', async () => {
|
|
376
|
+
artifacts.computed_users = {
|
|
377
|
+
connectionId: 'computed_users',
|
|
378
|
+
type: 'MongoDBCollection',
|
|
379
|
+
properties: { databaseUri: { _env: 'URI' }, collection: 'users' },
|
|
380
|
+
};
|
|
381
|
+
fs.writeFileSync(
|
|
382
|
+
path.join(buildDirectory, 'connections', 'computed_users.json'),
|
|
383
|
+
JSON.stringify(artifacts.computed_users)
|
|
384
|
+
);
|
|
385
|
+
fs.writeFileSync(
|
|
386
|
+
path.join(configDirectory, 'tests', 'data', 'computed.yaml'),
|
|
387
|
+
'fixtures:\n auth_users: [{ _id: auth }]\n computed_users: [{ _id: computed }]\n'
|
|
388
|
+
);
|
|
389
|
+
try {
|
|
390
|
+
const dataSet = await readDataSet({ configDirectory, buildDirectory, name: 'computed' });
|
|
391
|
+
const { session, close } = await openDataSession({ dataSet });
|
|
392
|
+
try {
|
|
393
|
+
// AUTH_MONGODB_URI sorts first, so its database is the session's first.
|
|
394
|
+
const authDb = await redirectedDb({ session, connectionId: 'auth_users' });
|
|
395
|
+
const computedDb = await redirectedDb({ session, connectionId: 'computed_users' });
|
|
396
|
+
expect(authDb.databaseName).toEqual(session.databaseName);
|
|
397
|
+
expect(computedDb.databaseName).toEqual(session.databaseName);
|
|
398
|
+
const ids = await authDb.collection('users').distinct('_id');
|
|
399
|
+
expect(ids.sort()).toEqual(['auth', 'computed']);
|
|
400
|
+
} finally {
|
|
401
|
+
await close();
|
|
402
|
+
}
|
|
403
|
+
} finally {
|
|
404
|
+
delete artifacts.computed_users;
|
|
405
|
+
}
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
test('a session database the gate numbers after open survives a concurrent sweep and drops on close()', async () => {
|
|
409
|
+
const dataSet = await readDataSet({ configDirectory, buildDirectory, name: 'targets' });
|
|
410
|
+
const first = await openDataSession({ dataSet });
|
|
411
|
+
// A target the build did not name, as a config mutant's would be.
|
|
412
|
+
artifacts.late = {
|
|
413
|
+
connectionId: 'late',
|
|
414
|
+
type: 'MongoDBCollection',
|
|
415
|
+
properties: { databaseUri: { _secret: 'LATE_URI' }, collection: 'users' },
|
|
416
|
+
};
|
|
417
|
+
let lateName;
|
|
418
|
+
try {
|
|
419
|
+
const db = await redirectedDb({ session: first.session, connectionId: 'late' });
|
|
420
|
+
await db.collection('users').insertOne({ _id: 'late' });
|
|
421
|
+
lateName = db.databaseName;
|
|
422
|
+
} finally {
|
|
423
|
+
delete artifacts.late;
|
|
424
|
+
}
|
|
425
|
+
expect(lateName).toEqual(`${first.session.prefix}_${dataSet.databaseTargets.length}`);
|
|
426
|
+
const second = await openDataSession({ dataSet });
|
|
427
|
+
expect(await listDatabaseNames()).toContain(lateName);
|
|
428
|
+
await second.close();
|
|
429
|
+
await first.close();
|
|
430
|
+
expect(await listDatabaseNames()).not.toContain(lateName);
|
|
431
|
+
});
|
|
432
|
+
});
|
|
433
|
+
|
|
284
434
|
test('close() waits for background work added to the session', async () => {
|
|
285
435
|
const { session, close } = await openDataSession({ dataSet: makeDataSet({}) });
|
|
286
436
|
let release;
|
|
@@ -14,10 +14,11 @@
|
|
|
14
14
|
limitations under the License.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
+
import { type } from '@lowdefy/helpers';
|
|
17
18
|
import { parseDataSet } from '@lowdefy/node-utils';
|
|
18
19
|
|
|
19
20
|
import checkSharedCollectionIds from './checkSharedCollectionIds.js';
|
|
20
|
-
import
|
|
21
|
+
import describeTargetKey from './describeTargetKey.js';
|
|
21
22
|
import readConnectionArtifacts from './readConnectionArtifacts.js';
|
|
22
23
|
import resolveDataSetCollection from './resolveDataSetCollection.js';
|
|
23
24
|
|
|
@@ -31,38 +32,52 @@ function getKeyedConnectionIds({ dataSet }) {
|
|
|
31
32
|
];
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
// Sorts a URI's default database (no databaseName) before its named ones.
|
|
36
|
+
function sortKey(key) {
|
|
37
|
+
const [uri, databaseName] = JSON.parse(key);
|
|
38
|
+
return [uri, databaseName ?? ''].join('\u0000');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Every database target the built connections name, sorted, so each run numbers them alike and the
|
|
42
|
+
// first one (the database a computed target reads) is the same on every run.
|
|
43
|
+
function listDatabaseTargets({ artifacts }) {
|
|
44
|
+
const keys = new Set();
|
|
45
|
+
Object.values(artifacts).forEach((artifact) => {
|
|
46
|
+
const key = describeTargetKey({ properties: artifact.properties });
|
|
47
|
+
if (!type.isNone(key)) keys.add(key);
|
|
48
|
+
});
|
|
49
|
+
return [...keys].sort((a, b) => sortKey(a).localeCompare(sortKey(b)));
|
|
50
|
+
}
|
|
51
|
+
|
|
34
52
|
// The data set checks only the dev build can make, on top of parseDataSet's: every connection the
|
|
35
|
-
// data set names is a MongoDBCollection with a literal collection, and
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
53
|
+
// data set names is a MongoDBCollection with a literal collection, and a generated _id that another
|
|
54
|
+
// connection on the same collection in the same database target also holds is refused. Adds the
|
|
55
|
+
// database target of each connection it names (targets) and of every connection in the build
|
|
56
|
+
// (databaseTargets): a data session gives each target a database of its own. Runs before any
|
|
57
|
+
// browser opens.
|
|
39
58
|
async function readDataSet({ configDirectory, buildDirectory, name }) {
|
|
40
59
|
const dataSet = await parseDataSet({ configDirectory, name });
|
|
41
|
-
const warnings = [...dataSet.warnings];
|
|
42
|
-
|
|
43
60
|
const artifacts = await readConnectionArtifacts({ buildDirectory });
|
|
44
61
|
const keyedIds = getKeyedConnectionIds({ dataSet });
|
|
62
|
+
const databaseTargets = listDatabaseTargets({ artifacts });
|
|
45
63
|
const collections = {};
|
|
64
|
+
const targets = {};
|
|
46
65
|
keyedIds.forEach((connectionId) => {
|
|
66
|
+
const artifact = artifacts[connectionId];
|
|
47
67
|
collections[connectionId] = resolveDataSetCollection({
|
|
48
68
|
dataSetName: name,
|
|
49
69
|
connectionId,
|
|
50
|
-
artifact
|
|
70
|
+
artifact,
|
|
51
71
|
}).collection;
|
|
72
|
+
// A target computed per request reads the session's first database, the first target's, so it
|
|
73
|
+
// groups and checks _ids with that target's connections.
|
|
74
|
+
targets[connectionId] =
|
|
75
|
+
describeTargetKey({ properties: artifact.properties }) ?? databaseTargets[0] ?? null;
|
|
52
76
|
});
|
|
53
77
|
|
|
54
|
-
|
|
55
|
-
const [a, b] = connectionIds;
|
|
56
|
-
const message = `Connections "${a}" and "${b}" both name collection "${collection}" in different databases; under a data set they read one database, so they share one collection.`;
|
|
57
|
-
if (keyedIds.includes(a) || keyedIds.includes(b)) {
|
|
58
|
-
throw new Error(`Data set "${name}": ${message}`);
|
|
59
|
-
}
|
|
60
|
-
warnings.push(message);
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
checkSharedCollectionIds({ dataSetName: name, dataSet: { ...dataSet, collections } });
|
|
78
|
+
checkSharedCollectionIds({ dataSetName: name, dataSet: { ...dataSet, collections, targets } });
|
|
64
79
|
|
|
65
|
-
return { ...dataSet, collections,
|
|
80
|
+
return { ...dataSet, collections, targets, databaseTargets };
|
|
66
81
|
}
|
|
67
82
|
|
|
68
83
|
export default readDataSet;
|