abmp-npm 2.0.69 → 2.0.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend/__tests__/daily-pull-execution-check.test.js +124 -0
- package/backend/__tests__/transient-retry-and-bulk-lookup.test.js +129 -0
- package/backend/daily-pull/bulk-process-methods.js +9 -1
- package/backend/daily-pull/process-member-methods.js +12 -2
- package/backend/daily-pull/schedule-methods.js +41 -0
- package/backend/jobs.js +12 -18
- package/backend/members-data-methods.js +50 -5
- package/backend/tasks/daily-pull-check-methods.js +38 -18
- package/backend/utils.js +42 -0
- package/package.json +2 -1
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
jest.mock('../elevated-modules', () => ({
|
|
2
|
+
wixData: { query: jest.fn() },
|
|
3
|
+
}));
|
|
4
|
+
jest.mock('psdev-task-manager', () => ({ taskManager: jest.fn() }));
|
|
5
|
+
|
|
6
|
+
const { taskManager } = require('psdev-task-manager');
|
|
7
|
+
|
|
8
|
+
const { buildDailyPullTasks } = require('../daily-pull/schedule-methods');
|
|
9
|
+
const { wixData } = require('../elevated-modules');
|
|
10
|
+
const { TASKS_NAMES } = require('../tasks/consts');
|
|
11
|
+
const { dailyPullExecutionCheck } = require('../tasks/daily-pull-check-methods');
|
|
12
|
+
|
|
13
|
+
const makeQueryResult = items => ({ items, hasNext: () => false });
|
|
14
|
+
|
|
15
|
+
const mockTasksQuery = items => {
|
|
16
|
+
const query = {
|
|
17
|
+
hasSome: jest.fn().mockReturnThis(),
|
|
18
|
+
ge: jest.fn().mockReturnThis(),
|
|
19
|
+
find: jest.fn().mockResolvedValue(makeQueryResult(items)),
|
|
20
|
+
};
|
|
21
|
+
wixData.query.mockReturnValue(query);
|
|
22
|
+
return query;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
describe('buildDailyPullTasks', () => {
|
|
26
|
+
test('schedules one ScheduleMembersDataPerAction task per action, excluding none by default', () => {
|
|
27
|
+
const tasks = buildDailyPullTasks();
|
|
28
|
+
|
|
29
|
+
expect(tasks.map(task => task.data.action).sort()).toEqual(['drop', 'new', 'update']);
|
|
30
|
+
tasks.forEach(task => {
|
|
31
|
+
expect(task.name).toBe(TASKS_NAMES.ScheduleMembersDataPerAction);
|
|
32
|
+
expect(task.type).toBe('scheduled');
|
|
33
|
+
expect(task.data).toEqual({ action: task.data.action });
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('includes the none action and flag when includeNone is set', () => {
|
|
38
|
+
const tasks = buildDailyPullTasks({ includeNone: true });
|
|
39
|
+
|
|
40
|
+
expect(tasks.map(task => task.data.action).sort()).toEqual(['drop', 'new', 'none', 'update']);
|
|
41
|
+
tasks.forEach(task => expect(task.data.includeNone).toBe(true));
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test('propagates isTestEnvironment and backupDate to every task', () => {
|
|
45
|
+
const tasks = buildDailyPullTasks({ isTestEnvironment: true, backupDate: '2026-06-01' });
|
|
46
|
+
|
|
47
|
+
tasks.forEach(task => {
|
|
48
|
+
expect(task.data.isTestEnvironment).toBe(true);
|
|
49
|
+
expect(task.data.backupDate).toBe('2026-06-01');
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('dailyPullExecutionCheck', () => {
|
|
55
|
+
let scheduleInBulk;
|
|
56
|
+
|
|
57
|
+
beforeEach(() => {
|
|
58
|
+
wixData.query.mockReset();
|
|
59
|
+
scheduleInBulk = jest.fn().mockResolvedValue(undefined);
|
|
60
|
+
taskManager.mockReturnValue({ scheduleInBulk });
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('counts the per-action tasks the cron creates as evidence the pull ran', async () => {
|
|
64
|
+
const query = mockTasksQuery([{ name: TASKS_NAMES.ScheduleMembersDataPerAction }]);
|
|
65
|
+
|
|
66
|
+
const result = await dailyPullExecutionCheck({});
|
|
67
|
+
|
|
68
|
+
expect(query.hasSome).toHaveBeenCalledWith('name', [
|
|
69
|
+
TASKS_NAMES.ScheduleMembersDataPerAction,
|
|
70
|
+
TASKS_NAMES.ScheduleDailyMembersDataSync,
|
|
71
|
+
]);
|
|
72
|
+
expect(result.success).toBe(true);
|
|
73
|
+
expect(result.fallbackScheduled).toBeUndefined();
|
|
74
|
+
expect(scheduleInBulk).not.toHaveBeenCalled();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('schedules the per-action fallback with environment flags when no pull is found', async () => {
|
|
78
|
+
mockTasksQuery([]);
|
|
79
|
+
|
|
80
|
+
const result = await dailyPullExecutionCheck({ isTestEnvironment: true, includeNone: true });
|
|
81
|
+
|
|
82
|
+
expect(result.success).toBe(false);
|
|
83
|
+
expect(result.fallbackScheduled).toBe(true);
|
|
84
|
+
expect(scheduleInBulk).toHaveBeenCalledTimes(1);
|
|
85
|
+
const scheduledTasks = scheduleInBulk.mock.calls[0][0];
|
|
86
|
+
expect(scheduledTasks.map(task => task.data.action).sort()).toEqual([
|
|
87
|
+
'drop',
|
|
88
|
+
'new',
|
|
89
|
+
'none',
|
|
90
|
+
'update',
|
|
91
|
+
]);
|
|
92
|
+
scheduledTasks.forEach(task => {
|
|
93
|
+
expect(task.name).toBe(TASKS_NAMES.ScheduleMembersDataPerAction);
|
|
94
|
+
expect(task.data.isTestEnvironment).toBe(true);
|
|
95
|
+
expect(task.data.includeNone).toBe(true);
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('fallback defaults to production flags when no task data is provided', async () => {
|
|
100
|
+
mockTasksQuery([]);
|
|
101
|
+
|
|
102
|
+
await dailyPullExecutionCheck(undefined);
|
|
103
|
+
|
|
104
|
+
const scheduledTasks = scheduleInBulk.mock.calls[0][0];
|
|
105
|
+
expect(scheduledTasks.map(task => task.data.action).sort()).toEqual(['drop', 'new', 'update']);
|
|
106
|
+
scheduledTasks.forEach(task => {
|
|
107
|
+
expect(task.data.isTestEnvironment).toBeUndefined();
|
|
108
|
+
expect(task.data.includeNone).toBeUndefined();
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test('uses the provided lookback window', async () => {
|
|
113
|
+
const query = mockTasksQuery([{ name: TASKS_NAMES.ScheduleDailyMembersDataSync }]);
|
|
114
|
+
const before = Date.now();
|
|
115
|
+
|
|
116
|
+
const result = await dailyPullExecutionCheck({ hoursBack: 12 });
|
|
117
|
+
|
|
118
|
+
const sinceDate = query.ge.mock.calls[0][1];
|
|
119
|
+
expect(query.ge).toHaveBeenCalledWith('_createdDate', expect.any(Date));
|
|
120
|
+
const expectedSince = before - 12 * 60 * 60 * 1000;
|
|
121
|
+
expect(Math.abs(sinceDate.getTime() - expectedSince)).toBeLessThan(5000);
|
|
122
|
+
expect(result.success).toBe(true);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
jest.mock('../elevated-modules', () => ({
|
|
2
|
+
wixData: { query: jest.fn() },
|
|
3
|
+
}));
|
|
4
|
+
|
|
5
|
+
const { wixData } = require('../elevated-modules');
|
|
6
|
+
const { findMembersByIds } = require('../members-data-methods');
|
|
7
|
+
const { withTransientErrorRetry, isTransientNetworkError } = require('../utils');
|
|
8
|
+
|
|
9
|
+
const makeQueryResult = items => ({ items, hasNext: () => false });
|
|
10
|
+
|
|
11
|
+
const mockQueryReturning = find => {
|
|
12
|
+
const query = {
|
|
13
|
+
hasSome: jest.fn().mockReturnThis(),
|
|
14
|
+
limit: jest.fn().mockReturnThis(),
|
|
15
|
+
find,
|
|
16
|
+
};
|
|
17
|
+
wixData.query.mockReturnValue(query);
|
|
18
|
+
return query;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
describe('isTransientNetworkError', () => {
|
|
22
|
+
test('matches undici "fetch failed" and common network error codes', () => {
|
|
23
|
+
expect(isTransientNetworkError(new Error('fetch failed'))).toBe(true);
|
|
24
|
+
expect(isTransientNetworkError({ message: 'request failed', code: 'ECONNRESET' })).toBe(true);
|
|
25
|
+
expect(
|
|
26
|
+
isTransientNetworkError({ message: 'fetch failed', cause: { code: 'UND_ERR_SOCKET' } })
|
|
27
|
+
).toBe(true);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test('does not match application errors', () => {
|
|
31
|
+
expect(isTransientNetworkError(new Error('Multiple members found with memberId 1'))).toBe(
|
|
32
|
+
false
|
|
33
|
+
);
|
|
34
|
+
expect(isTransientNetworkError(new Error('WDE0025: validation failed'))).toBe(false);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
describe('withTransientErrorRetry', () => {
|
|
39
|
+
test('retries transient failures and resolves with the eventual result', async () => {
|
|
40
|
+
const operation = jest
|
|
41
|
+
.fn()
|
|
42
|
+
.mockRejectedValueOnce(new Error('fetch failed'))
|
|
43
|
+
.mockResolvedValueOnce('ok');
|
|
44
|
+
|
|
45
|
+
await expect(withTransientErrorRetry(operation, { baseDelayMs: 1 })).resolves.toBe('ok');
|
|
46
|
+
expect(operation).toHaveBeenCalledTimes(2);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('rethrows immediately on non-transient errors', async () => {
|
|
50
|
+
const operation = jest.fn().mockRejectedValue(new Error('validation failed'));
|
|
51
|
+
|
|
52
|
+
await expect(withTransientErrorRetry(operation, { baseDelayMs: 1 })).rejects.toThrow(
|
|
53
|
+
'validation failed'
|
|
54
|
+
);
|
|
55
|
+
expect(operation).toHaveBeenCalledTimes(1);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('gives up after the configured number of retries', async () => {
|
|
59
|
+
const operation = jest.fn().mockRejectedValue(new Error('fetch failed'));
|
|
60
|
+
|
|
61
|
+
await expect(
|
|
62
|
+
withTransientErrorRetry(operation, { retries: 2, baseDelayMs: 1 })
|
|
63
|
+
).rejects.toThrow('fetch failed');
|
|
64
|
+
expect(operation).toHaveBeenCalledTimes(3);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
describe('findMembersByIds', () => {
|
|
69
|
+
beforeEach(() => {
|
|
70
|
+
wixData.query.mockReset();
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test('returns a map of String(memberId) to member record', async () => {
|
|
74
|
+
mockQueryReturning(
|
|
75
|
+
jest.fn().mockResolvedValue(
|
|
76
|
+
makeQueryResult([
|
|
77
|
+
{ _id: 'a', memberId: 1 },
|
|
78
|
+
{ _id: 'b', memberId: 2 },
|
|
79
|
+
])
|
|
80
|
+
)
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
const membersById = await findMembersByIds([1, 2, 3]);
|
|
84
|
+
|
|
85
|
+
expect(membersById.get('1')).toEqual({ _id: 'a', memberId: 1 });
|
|
86
|
+
expect(membersById.get('2')).toEqual({ _id: 'b', memberId: 2 });
|
|
87
|
+
expect(membersById.has('3')).toBe(false);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('deduplicates input IDs and skips null/undefined before querying', async () => {
|
|
91
|
+
const query = mockQueryReturning(jest.fn().mockResolvedValue(makeQueryResult([])));
|
|
92
|
+
|
|
93
|
+
await findMembersByIds([1, 1, null, undefined, 2]);
|
|
94
|
+
|
|
95
|
+
expect(query.hasSome).toHaveBeenCalledWith('memberId', [1, 2]);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test('returns an empty map without querying when there are no IDs', async () => {
|
|
99
|
+
const membersById = await findMembersByIds([]);
|
|
100
|
+
|
|
101
|
+
expect(membersById.size).toBe(0);
|
|
102
|
+
expect(wixData.query).not.toHaveBeenCalled();
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test('chunks large ID lists into multiple queries', async () => {
|
|
106
|
+
const query = mockQueryReturning(jest.fn().mockResolvedValue(makeQueryResult([])));
|
|
107
|
+
const ids = Array.from({ length: 250 }, (_, i) => i + 1);
|
|
108
|
+
|
|
109
|
+
await findMembersByIds(ids);
|
|
110
|
+
|
|
111
|
+
expect(query.hasSome).toHaveBeenCalledTimes(3);
|
|
112
|
+
expect(query.hasSome.mock.calls.map(call => call[1].length)).toEqual([100, 100, 50]);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('throws when multiple records share the same memberId', async () => {
|
|
116
|
+
mockQueryReturning(
|
|
117
|
+
jest.fn().mockResolvedValue(
|
|
118
|
+
makeQueryResult([
|
|
119
|
+
{ _id: 'a', memberId: 1 },
|
|
120
|
+
{ _id: 'b', memberId: 1 },
|
|
121
|
+
])
|
|
122
|
+
)
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
await expect(findMembersByIds([1])).rejects.toThrow(
|
|
126
|
+
'Multiple members found with memberId(s): [1]'
|
|
127
|
+
);
|
|
128
|
+
});
|
|
129
|
+
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const { bulkSaveMembers, getMemberBySlug } = require('../members-data-methods');
|
|
1
|
+
const { bulkSaveMembers, getMemberBySlug, findMembersByIds } = require('../members-data-methods');
|
|
2
2
|
const { extractUrlCounter } = require('../utils');
|
|
3
3
|
|
|
4
4
|
const { generateUpdatedMemberData } = require('./process-member-methods');
|
|
@@ -132,10 +132,18 @@ const bulkProcessAndSaveMemberData = async ({ memberDataList, currentPageNumber
|
|
|
132
132
|
const startTime = Date.now();
|
|
133
133
|
|
|
134
134
|
try {
|
|
135
|
+
// Prefetch all existing members for this page in a few chunked queries instead of
|
|
136
|
+
// one query per member — thousands of parallel lookups made the whole page fail
|
|
137
|
+
// whenever a single one hit a transient network error.
|
|
138
|
+
const existingMembersById = await findMembersByIds(
|
|
139
|
+
memberDataList.map(memberData => memberData?.memberid)
|
|
140
|
+
);
|
|
141
|
+
|
|
135
142
|
const processedMemberDataPromises = memberDataList.map(memberData =>
|
|
136
143
|
generateUpdatedMemberData({
|
|
137
144
|
inputMemberData: memberData,
|
|
138
145
|
currentPageNumber,
|
|
146
|
+
existingDbMember: existingMembersById.get(String(memberData?.memberid)) ?? null,
|
|
139
147
|
})
|
|
140
148
|
);
|
|
141
149
|
|
|
@@ -58,16 +58,26 @@ const ensureUniqueUrl = async ({ url, memberId, fullName }) => {
|
|
|
58
58
|
* @param {Object} options - The options object
|
|
59
59
|
* @param {Object} options.inputMemberData - Raw member data from API
|
|
60
60
|
* @param {number} options.currentPageNumber - Current page number being processed
|
|
61
|
+
* @param {Object|null} [options.existingDbMember] - Prefetched DB member (null if none exists).
|
|
62
|
+
* Pass it when processing members in bulk to avoid one DB query per member; when omitted,
|
|
63
|
+
* the member is looked up individually.
|
|
61
64
|
* @returns {Promise<Object|null>} - Complete updated member data or null if validation fails
|
|
62
65
|
*/
|
|
63
|
-
async function generateUpdatedMemberData({
|
|
66
|
+
async function generateUpdatedMemberData({
|
|
67
|
+
inputMemberData,
|
|
68
|
+
currentPageNumber,
|
|
69
|
+
existingDbMember: prefetchedDbMember,
|
|
70
|
+
}) {
|
|
64
71
|
if (!validateCoreMemberData(inputMemberData)) {
|
|
65
72
|
throw new Error(
|
|
66
73
|
'Invalid member data: memberid, email (valid string), and memberships (array) are required'
|
|
67
74
|
);
|
|
68
75
|
}
|
|
69
76
|
|
|
70
|
-
const existingDbMember =
|
|
77
|
+
const existingDbMember =
|
|
78
|
+
prefetchedDbMember === undefined
|
|
79
|
+
? await findMemberById(inputMemberData.memberid)
|
|
80
|
+
: prefetchedDbMember;
|
|
71
81
|
|
|
72
82
|
const updatedMemberData = await createCoreMemberData(
|
|
73
83
|
inputMemberData,
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
const { taskManager } = require('psdev-task-manager');
|
|
2
|
+
|
|
3
|
+
const { TASKS_NAMES } = require('../tasks/consts');
|
|
4
|
+
|
|
5
|
+
const { MEMBER_ACTIONS } = require('./consts');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Builds the per-action daily pull tasks. Single source of truth for what a
|
|
9
|
+
* "daily pull" schedules, shared by the cron job and the execution-check
|
|
10
|
+
* fallback so both produce identical tasks (same names, same environment flags).
|
|
11
|
+
* @param {Object} [options]
|
|
12
|
+
* @param {string} [options.backupDate] - Optional backup date (YYYY-MM-DD) to pull from the backup endpoint
|
|
13
|
+
* @param {boolean} [options.isTestEnvironment=false] - Whether to pull from the test PAC API
|
|
14
|
+
* @param {boolean} [options.includeNone=false] - Whether to also sync the NONE action
|
|
15
|
+
* @returns {Array<Object>} - Task definitions for taskManager().scheduleInBulk
|
|
16
|
+
*/
|
|
17
|
+
const buildDailyPullTasks = ({ backupDate, isTestEnvironment, includeNone } = {}) => {
|
|
18
|
+
const actionsToSync = includeNone
|
|
19
|
+
? Object.values(MEMBER_ACTIONS)
|
|
20
|
+
: Object.values(MEMBER_ACTIONS).filter(action => action !== MEMBER_ACTIONS.NONE);
|
|
21
|
+
return actionsToSync.map(action => ({
|
|
22
|
+
name: TASKS_NAMES.ScheduleMembersDataPerAction,
|
|
23
|
+
data: {
|
|
24
|
+
action,
|
|
25
|
+
...(backupDate ? { backupDate } : {}),
|
|
26
|
+
...(isTestEnvironment ? { isTestEnvironment } : {}),
|
|
27
|
+
...(includeNone ? { includeNone } : {}),
|
|
28
|
+
},
|
|
29
|
+
type: 'scheduled',
|
|
30
|
+
}));
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Schedules the daily pull (one ScheduleMembersDataPerAction task per action).
|
|
35
|
+
* @param {Object} [options] - Same options as buildDailyPullTasks
|
|
36
|
+
* @returns {Promise<any>}
|
|
37
|
+
*/
|
|
38
|
+
const scheduleDailyPullTasks = options =>
|
|
39
|
+
taskManager().scheduleInBulk(buildDailyPullTasks(options));
|
|
40
|
+
|
|
41
|
+
module.exports = { buildDailyPullTasks, scheduleDailyPullTasks };
|
package/backend/jobs.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const { taskManager } = require('psdev-task-manager');
|
|
2
2
|
|
|
3
|
-
const {
|
|
3
|
+
const { scheduleDailyPullTasks } = require('./daily-pull/schedule-methods');
|
|
4
4
|
const { TASKS_NAMES } = require('./tasks/consts');
|
|
5
5
|
const { dailyPullExecutionCheck } = require('./tasks/daily-pull-check-methods');
|
|
6
6
|
const { TASKS } = require('./tasks/tasks-configs');
|
|
@@ -34,21 +34,7 @@ async function scheduleDailyPullTask(options = null) {
|
|
|
34
34
|
console.log(`isTestEnvironment: ${isTestEnvironment}`);
|
|
35
35
|
console.log(`includeNone: ${includeNone}`);
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
? Object.values(MEMBER_ACTIONS)
|
|
39
|
-
: Object.values(MEMBER_ACTIONS).filter(action => action !== MEMBER_ACTIONS.NONE);
|
|
40
|
-
const toScheduleTasks = actionsToSync.map(action => ({
|
|
41
|
-
name: TASKS_NAMES.ScheduleMembersDataPerAction,
|
|
42
|
-
data: {
|
|
43
|
-
action,
|
|
44
|
-
...(backupDate ? { backupDate } : {}),
|
|
45
|
-
...(isTestEnvironment ? { isTestEnvironment } : {}),
|
|
46
|
-
...(includeNone ? { includeNone } : {}),
|
|
47
|
-
},
|
|
48
|
-
type: 'scheduled',
|
|
49
|
-
}));
|
|
50
|
-
|
|
51
|
-
return await taskManager().scheduleInBulk(toScheduleTasks);
|
|
37
|
+
return await scheduleDailyPullTasks({ backupDate, isTestEnvironment, includeNone });
|
|
52
38
|
} catch (error) {
|
|
53
39
|
console.error(`Failed to scheduleDailyPullTask: ${error.message}`);
|
|
54
40
|
throw new Error(`Failed to scheduleDailyPullTask: ${error.message}`);
|
|
@@ -111,10 +97,18 @@ async function scheduleNormalizeMemberEmailsTask() {
|
|
|
111
97
|
}
|
|
112
98
|
}
|
|
113
99
|
|
|
114
|
-
|
|
100
|
+
/**
|
|
101
|
+
* Runs the daily pull execution check (watchdog).
|
|
102
|
+
* @param {Object} [options]
|
|
103
|
+
* @param {number} [options.hoursBack=4] - Lookback window in hours
|
|
104
|
+
* @param {boolean} [options.isTestEnvironment=false] - Pull from the test PAC API if the fallback fires
|
|
105
|
+
* @param {boolean} [options.includeNone=false] - Include the NONE action if the fallback fires
|
|
106
|
+
* @returns {Promise<Object>} - Check result
|
|
107
|
+
*/
|
|
108
|
+
async function runDailyPullExecutionCheck(options = {}) {
|
|
115
109
|
try {
|
|
116
110
|
console.log('runDailyPullExecutionCheck started!');
|
|
117
|
-
return await dailyPullExecutionCheck({});
|
|
111
|
+
return await dailyPullExecutionCheck(options || {});
|
|
118
112
|
} catch (error) {
|
|
119
113
|
console.error(`Failed to runDailyPullExecutionCheck: ${error.message}`);
|
|
120
114
|
throw new Error(`Failed to runDailyPullExecutionCheck: ${error.message}`);
|
|
@@ -15,6 +15,7 @@ const {
|
|
|
15
15
|
generateGeoHash,
|
|
16
16
|
searchAllItems,
|
|
17
17
|
runIf,
|
|
18
|
+
withTransientErrorRetry,
|
|
18
19
|
} = require('./utils');
|
|
19
20
|
|
|
20
21
|
/**
|
|
@@ -158,11 +159,9 @@ async function findMemberById(memberId) {
|
|
|
158
159
|
}
|
|
159
160
|
|
|
160
161
|
try {
|
|
161
|
-
const queryResult = await
|
|
162
|
-
.query(COLLECTIONS.MEMBERS_DATA)
|
|
163
|
-
|
|
164
|
-
.limit(2)
|
|
165
|
-
.find();
|
|
162
|
+
const queryResult = await withTransientErrorRetry(() =>
|
|
163
|
+
wixData.query(COLLECTIONS.MEMBERS_DATA).eq('memberId', memberId).limit(2).find()
|
|
164
|
+
);
|
|
166
165
|
if (queryResult.items.length > 1) {
|
|
167
166
|
throw new Error(
|
|
168
167
|
`Multiple members found with memberId ${memberId} members _ids are : [${queryResult.items.map(member => member._id).join(', ')}]`
|
|
@@ -175,6 +174,51 @@ async function findMemberById(memberId) {
|
|
|
175
174
|
}
|
|
176
175
|
}
|
|
177
176
|
|
|
177
|
+
/**
|
|
178
|
+
* Retrieves existing members for a list of member IDs in bulk.
|
|
179
|
+
* Uses chunked `hasSome` queries so a full page of members costs a handful of
|
|
180
|
+
* requests instead of one query per member (the per-member fan-out made a whole
|
|
181
|
+
* page fail whenever a single query hit a transient "fetch failed").
|
|
182
|
+
* @param {Array<string|number>} memberIds - Member IDs to look up
|
|
183
|
+
* @returns {Promise<Map<string, Object>>} - Map of String(memberId) to member record (missing IDs are absent)
|
|
184
|
+
*/
|
|
185
|
+
async function findMembersByIds(memberIds) {
|
|
186
|
+
const uniqueIds = [...new Set((memberIds || []).filter(id => id !== undefined && id !== null))];
|
|
187
|
+
const membersById = new Map();
|
|
188
|
+
if (uniqueIds.length === 0) {
|
|
189
|
+
return membersById;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
try {
|
|
193
|
+
const idChunks = chunkArray(uniqueIds, 100);
|
|
194
|
+
const chunkResults = await Promise.all(
|
|
195
|
+
idChunks.map(idsChunk =>
|
|
196
|
+
withTransientErrorRetry(() =>
|
|
197
|
+
queryAllItems(
|
|
198
|
+
wixData.query(COLLECTIONS.MEMBERS_DATA).hasSome('memberId', idsChunk).limit(1000)
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
const duplicateIds = new Set();
|
|
205
|
+
chunkResults.flat().forEach(member => {
|
|
206
|
+
const key = String(member.memberId);
|
|
207
|
+
if (membersById.has(key)) {
|
|
208
|
+
duplicateIds.add(key);
|
|
209
|
+
}
|
|
210
|
+
membersById.set(key, member);
|
|
211
|
+
});
|
|
212
|
+
if (duplicateIds.size > 0) {
|
|
213
|
+
throw new Error(`Multiple members found with memberId(s): [${[...duplicateIds].join(', ')}]`);
|
|
214
|
+
}
|
|
215
|
+
return membersById;
|
|
216
|
+
} catch (error) {
|
|
217
|
+
console.error('Error finding members by IDs:', error);
|
|
218
|
+
throw new Error(`Failed to retrieve member data: ${error.message}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
178
222
|
/**
|
|
179
223
|
* Method to get member by slug with flexible filtering options
|
|
180
224
|
* @param {Object} options - Query options
|
|
@@ -723,6 +767,7 @@ module.exports = {
|
|
|
723
767
|
saveRegistrationData,
|
|
724
768
|
bulkSaveMembers,
|
|
725
769
|
findMemberById,
|
|
770
|
+
findMembersByIds,
|
|
726
771
|
getMemberBySlug,
|
|
727
772
|
getCMSMemberByWixMemberId,
|
|
728
773
|
getAllEmptyAboutYouMembers,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const { taskManager } = require('psdev-task-manager');
|
|
2
1
|
const { COLLECTIONS } = require('psdev-task-manager/public/consts');
|
|
3
2
|
|
|
3
|
+
const { scheduleDailyPullTasks } = require('../daily-pull/schedule-methods');
|
|
4
4
|
const { wixData } = require('../elevated-modules');
|
|
5
5
|
const { queryAllItems } = require('../utils');
|
|
6
6
|
|
|
@@ -8,43 +8,63 @@ const { TASKS_NAMES } = require('./consts');
|
|
|
8
8
|
|
|
9
9
|
const DEFAULT_HOURS_BACK = 4;
|
|
10
10
|
|
|
11
|
+
// The cron path (scheduleDailyPullTask) creates ScheduleMembersDataPerAction tasks
|
|
12
|
+
// directly; ScheduleDailyMembersDataSync only exists when the pull is triggered via
|
|
13
|
+
// the root task. Either one in the window is evidence the daily pull was scheduled.
|
|
14
|
+
const DAILY_PULL_TASK_NAMES = [
|
|
15
|
+
TASKS_NAMES.ScheduleMembersDataPerAction,
|
|
16
|
+
TASKS_NAMES.ScheduleDailyMembersDataSync,
|
|
17
|
+
];
|
|
18
|
+
|
|
11
19
|
/**
|
|
12
|
-
* Detects whether the daily pull was scheduled
|
|
13
|
-
* If no
|
|
20
|
+
* Detects whether the daily pull was scheduled in the lookback window.
|
|
21
|
+
* If no daily pull task exists, re-schedules the same per-action tasks the cron
|
|
22
|
+
* would have created, propagating the environment flags from the task data so a
|
|
23
|
+
* test site's fallback pulls from the test PAC API.
|
|
24
|
+
* @param {Object} [taskData]
|
|
25
|
+
* @param {number} [taskData.hoursBack=4] - Lookback window in hours
|
|
26
|
+
* @param {boolean} [taskData.isTestEnvironment=false] - Pull from the test PAC API on fallback
|
|
27
|
+
* @param {boolean} [taskData.includeNone=false] - Include the NONE action on fallback
|
|
28
|
+
* @returns {Promise<Object>} - Check result
|
|
14
29
|
*/
|
|
15
30
|
async function dailyPullExecutionCheck(taskData) {
|
|
16
31
|
const hoursBack =
|
|
17
32
|
taskData?.hoursBack && Number.isFinite(taskData.hoursBack)
|
|
18
33
|
? taskData.hoursBack
|
|
19
34
|
: DEFAULT_HOURS_BACK;
|
|
35
|
+
const isTestEnvironment = Boolean(taskData?.isTestEnvironment);
|
|
36
|
+
const includeNone = Boolean(taskData?.includeNone);
|
|
20
37
|
const sinceDate = new Date(Date.now() - hoursBack * 60 * 60 * 1000);
|
|
21
38
|
|
|
22
|
-
console.log('dailyPullExecutionCheck started', {
|
|
39
|
+
console.log('dailyPullExecutionCheck started', {
|
|
40
|
+
hoursBack,
|
|
41
|
+
sinceDate,
|
|
42
|
+
isTestEnvironment,
|
|
43
|
+
includeNone,
|
|
44
|
+
});
|
|
23
45
|
|
|
24
|
-
const
|
|
46
|
+
const dailyPullTasksQuery = wixData
|
|
25
47
|
.query(COLLECTIONS.TASKS)
|
|
26
|
-
.
|
|
48
|
+
.hasSome('name', DAILY_PULL_TASK_NAMES)
|
|
27
49
|
.ge('_createdDate', sinceDate);
|
|
28
50
|
|
|
29
|
-
const
|
|
30
|
-
const
|
|
51
|
+
const dailyPullTasks = await queryAllItems(dailyPullTasksQuery);
|
|
52
|
+
const dailyPullScheduled = dailyPullTasks.length > 0;
|
|
31
53
|
|
|
32
54
|
const result = {
|
|
33
|
-
success:
|
|
55
|
+
success: dailyPullScheduled,
|
|
34
56
|
sinceDate: sinceDate.toISOString(),
|
|
35
|
-
|
|
36
|
-
|
|
57
|
+
checkedTaskNames: DAILY_PULL_TASK_NAMES,
|
|
58
|
+
dailyPullTasksFound: dailyPullTasks.length,
|
|
37
59
|
};
|
|
38
60
|
|
|
39
|
-
if (!
|
|
40
|
-
console.log('
|
|
61
|
+
if (!dailyPullScheduled) {
|
|
62
|
+
console.log('No daily pull tasks found in window; re-scheduling per-action pull tasks', {
|
|
41
63
|
hoursBack,
|
|
64
|
+
isTestEnvironment,
|
|
65
|
+
includeNone,
|
|
42
66
|
});
|
|
43
|
-
await
|
|
44
|
-
name: TASKS_NAMES.ScheduleDailyMembersDataSync,
|
|
45
|
-
data: {},
|
|
46
|
-
type: 'scheduled',
|
|
47
|
-
});
|
|
67
|
+
await scheduleDailyPullTasks({ isTestEnvironment, includeNone });
|
|
48
68
|
result.fallbackScheduled = true;
|
|
49
69
|
}
|
|
50
70
|
|
package/backend/utils.js
CHANGED
|
@@ -210,6 +210,46 @@ function formatDateOnly(dateStr) {
|
|
|
210
210
|
|
|
211
211
|
const runIf = (condition, asyncFn) => (condition ? asyncFn() : Promise.resolve(null));
|
|
212
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Whether an error looks like a transient network failure that is safe to retry,
|
|
215
|
+
* e.g. undici's generic "fetch failed" thrown by the Wix SDKs, connection resets
|
|
216
|
+
* or DNS hiccups under heavy load.
|
|
217
|
+
* @param {Error} error
|
|
218
|
+
* @returns {boolean}
|
|
219
|
+
*/
|
|
220
|
+
const isTransientNetworkError = error => {
|
|
221
|
+
const message = `${error?.message || ''} ${error?.cause?.message || ''} ${error?.code || ''} ${error?.cause?.code || ''}`;
|
|
222
|
+
return /fetch failed|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EAI_AGAIN|EPIPE|UND_ERR|socket hang up|network error/i.test(
|
|
223
|
+
message
|
|
224
|
+
);
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Runs an async operation, retrying with exponential backoff when it fails with a
|
|
229
|
+
* transient network error. Non-transient errors are rethrown immediately.
|
|
230
|
+
* @param {Function} operation - Async function to run
|
|
231
|
+
* @param {Object} [options]
|
|
232
|
+
* @param {number} [options.retries=2] - Maximum number of retries after the first attempt
|
|
233
|
+
* @param {number} [options.baseDelayMs=500] - Delay before the first retry, doubled each retry
|
|
234
|
+
* @returns {Promise<any>} - The operation's resolved value
|
|
235
|
+
*/
|
|
236
|
+
const withTransientErrorRetry = async (operation, { retries = 2, baseDelayMs = 500 } = {}) => {
|
|
237
|
+
for (let attempt = 0; ; attempt++) {
|
|
238
|
+
try {
|
|
239
|
+
return await operation();
|
|
240
|
+
} catch (error) {
|
|
241
|
+
if (attempt >= retries || !isTransientNetworkError(error)) {
|
|
242
|
+
throw error;
|
|
243
|
+
}
|
|
244
|
+
const delayMs = baseDelayMs * 2 ** attempt;
|
|
245
|
+
console.warn(
|
|
246
|
+
`Transient network error (attempt ${attempt + 1}/${retries + 1}): ${error.message}. Retrying in ${delayMs}ms`
|
|
247
|
+
);
|
|
248
|
+
await new Promise(resolve => setTimeout(resolve, delayMs));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
|
|
213
253
|
module.exports = {
|
|
214
254
|
getSiteConfigs,
|
|
215
255
|
retrieveAllItems,
|
|
@@ -232,4 +272,6 @@ module.exports = {
|
|
|
232
272
|
isPAC_STAFF,
|
|
233
273
|
searchAllItems,
|
|
234
274
|
runIf,
|
|
275
|
+
isTransientNetworkError,
|
|
276
|
+
withTransientErrorRetry,
|
|
235
277
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "abmp-npm",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.72",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"check-cycles": "madge --circular .",
|
|
@@ -49,6 +49,7 @@
|
|
|
49
49
|
"axios": "^1.13.1",
|
|
50
50
|
"crypto": "^1.0.1",
|
|
51
51
|
"csv-parser": "^3.0.0",
|
|
52
|
+
"debug": "^4.4.3",
|
|
52
53
|
"jwt-js-decode": "^1.9.0",
|
|
53
54
|
"lodash": "^4.17.21",
|
|
54
55
|
"ngeohash": "^0.6.3",
|