abmp-npm 2.0.76 → 2.0.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -1
- package/pages/Home.js +27 -55
- package/pages/index.js +0 -1
- package/public/Utils/personalDetailsUtils.js +5 -1
- package/public/Utils/sharedUtils.js +32 -0
- package/public/consts.js +0 -1
- package/.claude/worktrees/fix-save-button-disable/.husky/pre-commit +0 -19
- package/.claude/worktrees/fix-save-button-disable/.prettierignore +0 -7
- package/.claude/worktrees/fix-save-button-disable/.prettierrc.json +0 -16
- package/.claude/worktrees/fix-save-button-disable/README.md +0 -1
- package/.claude/worktrees/fix-save-button-disable/backend/README-member-contact-flows.md +0 -136
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/daily-pull-execution-check.test.js +0 -124
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/ensure-unique-urls-in-batch.test.js +0 -129
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/login-email-sync.test.js +0 -49
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/transient-retry-and-bulk-lookup.test.js +0 -129
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/url-uniqueness.test.js +0 -194
- package/.claude/worktrees/fix-save-button-disable/backend/automations-methods.js +0 -13
- package/.claude/worktrees/fix-save-button-disable/backend/cms-data-methods.js +0 -251
- package/.claude/worktrees/fix-save-button-disable/backend/consts.js +0 -58
- package/.claude/worktrees/fix-save-button-disable/backend/contacts-methods.js +0 -67
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/bulk-process-methods.js +0 -222
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/consts.js +0 -39
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/index.js +0 -5
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/process-member-methods.js +0 -213
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/schedule-methods.js +0 -41
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/sync-to-cms-methods.js +0 -161
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/utils.js +0 -151
- package/.claude/worktrees/fix-save-button-disable/backend/data-hooks.js +0 -29
- package/.claude/worktrees/fix-save-button-disable/backend/dev-only-methods.js +0 -58
- package/.claude/worktrees/fix-save-button-disable/backend/elevated-modules.js +0 -18
- package/.claude/worktrees/fix-save-button-disable/backend/forms-methods.js +0 -52
- package/.claude/worktrees/fix-save-button-disable/backend/http-functions/httpFunctions.js +0 -86
- package/.claude/worktrees/fix-save-button-disable/backend/http-functions/index.js +0 -3
- package/.claude/worktrees/fix-save-button-disable/backend/http-functions/interests.js +0 -35
- package/.claude/worktrees/fix-save-button-disable/backend/index.js +0 -19
- package/.claude/worktrees/fix-save-button-disable/backend/jobs.js +0 -158
- package/.claude/worktrees/fix-save-button-disable/backend/login/generate-member-session-token.js +0 -27
- package/.claude/worktrees/fix-save-button-disable/backend/login/index.js +0 -8
- package/.claude/worktrees/fix-save-button-disable/backend/login/qa-login-methods.js +0 -54
- package/.claude/worktrees/fix-save-button-disable/backend/login/sso-methods.js +0 -159
- package/.claude/worktrees/fix-save-button-disable/backend/member-contact-orchestration.js +0 -111
- package/.claude/worktrees/fix-save-button-disable/backend/members-area-methods.js +0 -88
- package/.claude/worktrees/fix-save-button-disable/backend/members-data-methods.js +0 -789
- package/.claude/worktrees/fix-save-button-disable/backend/pac-api-methods.js +0 -54
- package/.claude/worktrees/fix-save-button-disable/backend/routers/index.js +0 -3
- package/.claude/worktrees/fix-save-button-disable/backend/routers/methods.js +0 -191
- package/.claude/worktrees/fix-save-button-disable/backend/routers/utils.js +0 -128
- package/.claude/worktrees/fix-save-button-disable/backend/search-filters-methods.js +0 -124
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/address-primary-methods.js +0 -256
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/address-visibility-methods.js +0 -160
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/consts.js +0 -34
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/daily-pull-check-methods.js +0 -78
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/email-normalize-methods.js +0 -134
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/index.js +0 -10
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/migration-methods.js +0 -46
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/tasks-configs.js +0 -251
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/tasks-helpers-methods.js +0 -433
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/tasks-process-methods.js +0 -606
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/url-migration-methods.js +0 -378
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/url-space-fix-methods.js +0 -174
- package/.claude/worktrees/fix-save-button-disable/backend/utils.js +0 -277
- package/.claude/worktrees/fix-save-button-disable/dev-only-scripts/extract-duplicate-url-groups.js +0 -201
- package/.claude/worktrees/fix-save-button-disable/dev-only-scripts/find-duplicate-ids.js +0 -159
- package/.claude/worktrees/fix-save-button-disable/dev-only-scripts/find-duplicate-urls.js +0 -201
- package/.claude/worktrees/fix-save-button-disable/eslint.config.js +0 -120
- package/.claude/worktrees/fix-save-button-disable/index.js +0 -5
- package/.claude/worktrees/fix-save-button-disable/package-lock.json +0 -12160
- package/.claude/worktrees/fix-save-button-disable/package.json +0 -60
- package/.claude/worktrees/fix-save-button-disable/pages/ContactUs.js +0 -129
- package/.claude/worktrees/fix-save-button-disable/pages/Home.js +0 -779
- package/.claude/worktrees/fix-save-button-disable/pages/LearnMore.js +0 -27
- package/.claude/worktrees/fix-save-button-disable/pages/LoadingPage.js +0 -29
- package/.claude/worktrees/fix-save-button-disable/pages/Profile.js +0 -354
- package/.claude/worktrees/fix-save-button-disable/pages/QAPage.js +0 -39
- package/.claude/worktrees/fix-save-button-disable/pages/SaveAlerts.js +0 -13
- package/.claude/worktrees/fix-save-button-disable/pages/SelectBannerImages.js +0 -46
- package/.claude/worktrees/fix-save-button-disable/pages/deleteConfirm.js +0 -19
- package/.claude/worktrees/fix-save-button-disable/pages/index.js +0 -12
- package/.claude/worktrees/fix-save-button-disable/pages/personalDetails.js +0 -2486
- package/.claude/worktrees/fix-save-button-disable/public/Utils/homePage.js +0 -798
- package/.claude/worktrees/fix-save-button-disable/public/Utils/personalDetailsUtils.js +0 -25
- package/.claude/worktrees/fix-save-button-disable/public/Utils/sharedUtils.js +0 -235
- package/.claude/worktrees/fix-save-button-disable/public/consts.js +0 -113
- package/.claude/worktrees/fix-save-button-disable/public/index.js +0 -5
- package/.claude/worktrees/fix-save-button-disable/public/messages.js +0 -16
- package/.claude/worktrees/fix-save-button-disable/public/sso-auth-methods.js +0 -43
- package/.husky/pre-commit +0 -19
- package/.prettierignore +0 -7
- package/.prettierrc.json +0 -16
- package/backend/__tests__/daily-pull-execution-check.test.js +0 -124
- package/backend/__tests__/ensure-unique-urls-in-batch.test.js +0 -129
- package/backend/__tests__/login-email-sync.test.js +0 -49
- package/backend/__tests__/transient-retry-and-bulk-lookup.test.js +0 -129
- package/backend/__tests__/url-uniqueness.test.js +0 -194
- package/dev-only-scripts/extract-duplicate-url-groups.js +0 -201
- package/dev-only-scripts/find-duplicate-ids.js +0 -159
- package/dev-only-scripts/find-duplicate-urls.js +0 -201
- package/eslint.config.js +0 -120
- package/pages/contactForLocation.js +0 -28
|
@@ -1,277 +0,0 @@
|
|
|
1
|
-
const { auth } = require('@wix/essentials');
|
|
2
|
-
const { secrets } = require('@wix/secrets');
|
|
3
|
-
const { site } = require('@wix/urls');
|
|
4
|
-
const { encode } = require('ngeohash');
|
|
5
|
-
|
|
6
|
-
const { COLLECTIONS, ADDRESS_STATUS_TYPES } = require('../public/consts');
|
|
7
|
-
const { formatAddress, generateId, findMainAddress } = require('../public/Utils/sharedUtils');
|
|
8
|
-
|
|
9
|
-
const { CONFIG_KEYS, GEO_HASH_PRECISION, MEMBERSHIPS_TYPES } = require('./consts');
|
|
10
|
-
const { wixData } = require('./elevated-modules');
|
|
11
|
-
const elevatedGetSecretValue = auth.elevate(secrets.getSecretValue);
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Retrieves site configuration values from the database
|
|
15
|
-
* @param {keyof typeof CONFIG_KEYS} [configKey] - The configuration key to retrieve
|
|
16
|
-
* @returns {Promise<any>} The configuration value for the specified key, or all configs if no key provided
|
|
17
|
-
* @example
|
|
18
|
-
* // Get specific config
|
|
19
|
-
* const emailTemplateId = await getSiteConfigs('AUTOMATION_EMAIL_TRIGGER_ID');
|
|
20
|
-
*
|
|
21
|
-
* // Get all configs
|
|
22
|
-
* const allConfigs = await getSiteConfigs();
|
|
23
|
-
*/
|
|
24
|
-
const getSiteConfigs = async configKey => {
|
|
25
|
-
if (configKey && !Object.values(CONFIG_KEYS).includes(configKey)) {
|
|
26
|
-
throw new Error(
|
|
27
|
-
`Invalid configKey: ${configKey}. Must be one of: ${Object.values(CONFIG_KEYS).join(', ')}`
|
|
28
|
-
);
|
|
29
|
-
}
|
|
30
|
-
const siteConfigs = await wixData.get(COLLECTIONS.SITE_CONFIGS, 'SINGLE_ITEM_ID');
|
|
31
|
-
if (configKey) {
|
|
32
|
-
return siteConfigs[configKey];
|
|
33
|
-
}
|
|
34
|
-
return siteConfigs;
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
const retrieveAllItems = async collectionName => {
|
|
38
|
-
let results = await wixData.query(collectionName).limit(1000).find();
|
|
39
|
-
let allItems = results.items;
|
|
40
|
-
while (results.hasNext()) {
|
|
41
|
-
results = await results.next();
|
|
42
|
-
allItems = allItems.concat(results.items);
|
|
43
|
-
}
|
|
44
|
-
return allItems;
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Format date to Month Year string
|
|
49
|
-
* @param {string} dateString - The date string to format
|
|
50
|
-
* @returns {string} Formatted date (e.g., "January 2024")
|
|
51
|
-
*/
|
|
52
|
-
function formatDateToMonthYear(dateString) {
|
|
53
|
-
if (!dateString) return '';
|
|
54
|
-
|
|
55
|
-
const date = new Date(dateString);
|
|
56
|
-
if (isNaN(date.getTime())) return '';
|
|
57
|
-
const options = { year: 'numeric', month: 'long' };
|
|
58
|
-
return date.toLocaleDateString('en-US', options);
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function hasStudentMembership({ member, checkAssociation = false, siteAssociation = null }) {
|
|
62
|
-
const memberships = member?.memberships;
|
|
63
|
-
if (!Array.isArray(memberships)) return false;
|
|
64
|
-
|
|
65
|
-
return memberships.some(membership => {
|
|
66
|
-
const isStudent = membership.membertype === MEMBERSHIPS_TYPES.STUDENT;
|
|
67
|
-
const hasCorrectAssociation = !checkAssociation || membership.association === siteAssociation;
|
|
68
|
-
return isStudent && hasCorrectAssociation;
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function isStudent(member) {
|
|
73
|
-
return hasStudentMembership({ member, checkAssociation: false });
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function isPAC_STAFF(member) {
|
|
77
|
-
return Boolean(
|
|
78
|
-
member?.memberships?.some(membership => membership.membertype === MEMBERSHIPS_TYPES.PAC_STAFF)
|
|
79
|
-
);
|
|
80
|
-
}
|
|
81
|
-
/**
|
|
82
|
-
* Get address display options for member
|
|
83
|
-
* @param {Object} member - The member object
|
|
84
|
-
* @returns {Array} Address display options
|
|
85
|
-
*/
|
|
86
|
-
function getAddressDisplayOptions(member) {
|
|
87
|
-
const addresses = member.addresses || [];
|
|
88
|
-
const displayOptions = member.addressDisplayOption || [];
|
|
89
|
-
if (addresses.length === 1 && addresses[0].key) {
|
|
90
|
-
return [{ key: addresses[0].key, isMain: true }];
|
|
91
|
-
}
|
|
92
|
-
return displayOptions;
|
|
93
|
-
}
|
|
94
|
-
function getMoreAddressesToDisplay(addresses = [], addressDisplayOption = []) {
|
|
95
|
-
const visible = addresses.filter(addr => addr.addressStatus !== ADDRESS_STATUS_TYPES.DONT_SHOW);
|
|
96
|
-
if (visible.length < 2) {
|
|
97
|
-
return [];
|
|
98
|
-
}
|
|
99
|
-
const mainAddress = findMainAddress(addressDisplayOption, addresses);
|
|
100
|
-
const remainingAddressesToFormat = mainAddress
|
|
101
|
-
? visible.filter(addr => addr?.key !== mainAddress.key)
|
|
102
|
-
: visible;
|
|
103
|
-
|
|
104
|
-
return remainingAddressesToFormat
|
|
105
|
-
.map(addr => {
|
|
106
|
-
const addressString = formatAddress(addr);
|
|
107
|
-
return addressString ? { _id: generateId(), address: addressString } : null;
|
|
108
|
-
})
|
|
109
|
-
.filter(Boolean);
|
|
110
|
-
}
|
|
111
|
-
const getAllItems = async querySearchResult => {
|
|
112
|
-
let oldResults = querySearchResult;
|
|
113
|
-
console.log(`found items: ${oldResults.items.length}`);
|
|
114
|
-
const allItems = oldResults.items;
|
|
115
|
-
while (oldResults.hasNext()) {
|
|
116
|
-
oldResults = await oldResults.next();
|
|
117
|
-
allItems.push(...oldResults.items);
|
|
118
|
-
}
|
|
119
|
-
console.log(`all items count : ${allItems.length}`);
|
|
120
|
-
return allItems;
|
|
121
|
-
};
|
|
122
|
-
const searchAllItems = async searchQuery => {
|
|
123
|
-
console.log('start search');
|
|
124
|
-
const searchResults = await searchQuery.run();
|
|
125
|
-
return getAllItems(searchResults);
|
|
126
|
-
};
|
|
127
|
-
|
|
128
|
-
const queryAllItems = async query => {
|
|
129
|
-
console.log('start query');
|
|
130
|
-
const queryResults = await query.find();
|
|
131
|
-
return getAllItems(queryResults);
|
|
132
|
-
};
|
|
133
|
-
/**
|
|
134
|
-
* Chunks large arrays into smaller chunks for processing
|
|
135
|
-
* @param {Array} array - Array to chunk
|
|
136
|
-
* @param {number} chunkSize - Size of each chunk
|
|
137
|
-
* @returns {Array} - Array of chunks
|
|
138
|
-
*/
|
|
139
|
-
const chunkArray = (array, chunkSize = 50) => {
|
|
140
|
-
const chunks = [];
|
|
141
|
-
for (let i = 0; i < array.length; i += chunkSize) {
|
|
142
|
-
chunks.push(array.slice(i, i + chunkSize));
|
|
143
|
-
}
|
|
144
|
-
return chunks;
|
|
145
|
-
};
|
|
146
|
-
|
|
147
|
-
const generateGeoHash = addresses => {
|
|
148
|
-
const geohash = addresses
|
|
149
|
-
?.filter(address => (isNaN(address?.latitude) && isNaN(address?.longitude) ? false : address))
|
|
150
|
-
?.map(address => encode(address.latitude, address.longitude, GEO_HASH_PRECISION));
|
|
151
|
-
return geohash && geohash.length > 0 ? geohash : [];
|
|
152
|
-
};
|
|
153
|
-
|
|
154
|
-
/**
|
|
155
|
-
* Validates if input is a non-empty array
|
|
156
|
-
* @param {*} input - Input to validate
|
|
157
|
-
* @returns {boolean} - True if input is a non-empty array
|
|
158
|
-
*/
|
|
159
|
-
const isValidArray = input => Array.isArray(input) && input.length > 0;
|
|
160
|
-
|
|
161
|
-
const normalizeUrlForComparison = url => {
|
|
162
|
-
if (!url) return url;
|
|
163
|
-
// Remove trailing pattern like "-1", "-2", etc.
|
|
164
|
-
return url.toLowerCase().replace(/-\d+$/, '');
|
|
165
|
-
};
|
|
166
|
-
|
|
167
|
-
const extractUrlCounter = url => {
|
|
168
|
-
if (!url) return -1;
|
|
169
|
-
const lastSegment = url.split('-').pop() || '0';
|
|
170
|
-
return /^\d+$/.test(lastSegment) ? parseInt(lastSegment, 10) : -1;
|
|
171
|
-
};
|
|
172
|
-
|
|
173
|
-
const sortByUrlCounterDescending = (a, b) => extractUrlCounter(b.url) - extractUrlCounter(a.url);
|
|
174
|
-
|
|
175
|
-
async function getSecret(secretKey) {
|
|
176
|
-
return (await elevatedGetSecretValue(secretKey)).value;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
async function getSiteBaseUrl() {
|
|
180
|
-
try {
|
|
181
|
-
const result = await site.listPublishedSiteUrls({
|
|
182
|
-
filters: { primary: true },
|
|
183
|
-
});
|
|
184
|
-
const baseUrl = result.urls[0].url;
|
|
185
|
-
if (!baseUrl) {
|
|
186
|
-
throw new Error('No Base URL Found');
|
|
187
|
-
}
|
|
188
|
-
return baseUrl;
|
|
189
|
-
} catch (error) {
|
|
190
|
-
throw new Error(`Failed to get site base URL: ${error?.message || error}`);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function encodeXml(value) {
|
|
195
|
-
if (!value) return '';
|
|
196
|
-
return (
|
|
197
|
-
String(value)
|
|
198
|
-
.replace(/&/g, '&')
|
|
199
|
-
.replace(/</g, '<')
|
|
200
|
-
.replace(/>/g, '>')
|
|
201
|
-
// eslint-disable-next-line no-useless-escape
|
|
202
|
-
.replace(/\"/g, '"')
|
|
203
|
-
.replace(/'/g, ''')
|
|
204
|
-
);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
function formatDateOnly(dateStr) {
|
|
208
|
-
return new Date(dateStr).toISOString().slice(0, 10);
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
const runIf = (condition, asyncFn) => (condition ? asyncFn() : Promise.resolve(null));
|
|
212
|
-
|
|
213
|
-
/**
|
|
214
|
-
* Whether an error looks like a transient network failure that is safe to retry,
|
|
215
|
-
* e.g. undici's generic "fetch failed" thrown by the Wix SDKs, connection resets
|
|
216
|
-
* or DNS hiccups under heavy load.
|
|
217
|
-
* @param {Error} error
|
|
218
|
-
* @returns {boolean}
|
|
219
|
-
*/
|
|
220
|
-
const isTransientNetworkError = error => {
|
|
221
|
-
const message = `${error?.message || ''} ${error?.cause?.message || ''} ${error?.code || ''} ${error?.cause?.code || ''}`;
|
|
222
|
-
return /fetch failed|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EAI_AGAIN|EPIPE|UND_ERR|socket hang up|network error/i.test(
|
|
223
|
-
message
|
|
224
|
-
);
|
|
225
|
-
};
|
|
226
|
-
|
|
227
|
-
/**
|
|
228
|
-
* Runs an async operation, retrying with exponential backoff when it fails with a
|
|
229
|
-
* transient network error. Non-transient errors are rethrown immediately.
|
|
230
|
-
* @param {Function} operation - Async function to run
|
|
231
|
-
* @param {Object} [options]
|
|
232
|
-
* @param {number} [options.retries=2] - Maximum number of retries after the first attempt
|
|
233
|
-
* @param {number} [options.baseDelayMs=500] - Delay before the first retry, doubled each retry
|
|
234
|
-
* @returns {Promise<any>} - The operation's resolved value
|
|
235
|
-
*/
|
|
236
|
-
const withTransientErrorRetry = async (operation, { retries = 2, baseDelayMs = 500 } = {}) => {
|
|
237
|
-
for (let attempt = 0; ; attempt++) {
|
|
238
|
-
try {
|
|
239
|
-
return await operation();
|
|
240
|
-
} catch (error) {
|
|
241
|
-
if (attempt >= retries || !isTransientNetworkError(error)) {
|
|
242
|
-
throw error;
|
|
243
|
-
}
|
|
244
|
-
const delayMs = baseDelayMs * 2 ** attempt;
|
|
245
|
-
console.warn(
|
|
246
|
-
`Transient network error (attempt ${attempt + 1}/${retries + 1}): ${error.message}. Retrying in ${delayMs}ms`
|
|
247
|
-
);
|
|
248
|
-
await new Promise(resolve => setTimeout(resolve, delayMs));
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
};
|
|
252
|
-
|
|
253
|
-
module.exports = {
|
|
254
|
-
getSiteConfigs,
|
|
255
|
-
retrieveAllItems,
|
|
256
|
-
chunkArray,
|
|
257
|
-
generateGeoHash,
|
|
258
|
-
isValidArray,
|
|
259
|
-
normalizeUrlForComparison,
|
|
260
|
-
sortByUrlCounterDescending,
|
|
261
|
-
extractUrlCounter,
|
|
262
|
-
queryAllItems,
|
|
263
|
-
formatDateToMonthYear,
|
|
264
|
-
isStudent,
|
|
265
|
-
hasStudentMembership,
|
|
266
|
-
getAddressDisplayOptions,
|
|
267
|
-
getSecret,
|
|
268
|
-
getSiteBaseUrl,
|
|
269
|
-
encodeXml,
|
|
270
|
-
formatDateOnly,
|
|
271
|
-
getMoreAddressesToDisplay,
|
|
272
|
-
isPAC_STAFF,
|
|
273
|
-
searchAllItems,
|
|
274
|
-
runIf,
|
|
275
|
-
isTransientNetworkError,
|
|
276
|
-
withTransientErrorRetry,
|
|
277
|
-
};
|
package/.claude/worktrees/fix-save-button-disable/dev-only-scripts/extract-duplicate-url-groups.js
DELETED
|
@@ -1,201 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
const csv = require('csv-parser');
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Reads a Members Data CSV and extracts groups of non-unique URLs:
|
|
8
|
-
* each group is a URL plus the list of IDs and memberIds that share that URL.
|
|
9
|
-
* Only outputs groups where the same URL appears more than once.
|
|
10
|
-
* Report includes duplicateUrls, memberIdsWithDuplicateUrls (flat array of unique memberIds
|
|
11
|
-
* in duplicate groups), and nonUniqueMemberIds (memberIds that appear more than once, with count).
|
|
12
|
-
*
|
|
13
|
-
* Usage: node dev-only-scripts/extract-duplicate-url-groups.js <path-to-csv> [output-json-path]
|
|
14
|
-
* Example: node dev-only-scripts/extract-duplicate-url-groups.js "/Users/Besan/Downloads/Members+Data+Latest (17).csv"
|
|
15
|
-
*
|
|
16
|
-
* CSV must have columns: "url", "ID", and "memberId" (case-insensitive).
|
|
17
|
-
*/
|
|
18
|
-
function extractDuplicateUrlGroups(csvFilePath, outputJsonPath) {
|
|
19
|
-
if (!csvFilePath) {
|
|
20
|
-
console.error('Error: CSV file path is required');
|
|
21
|
-
console.error(
|
|
22
|
-
'Usage: node dev-only-scripts/extract-duplicate-url-groups.js <path-to-csv> [output-json-path]'
|
|
23
|
-
);
|
|
24
|
-
process.exit(1);
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
if (!fs.existsSync(csvFilePath)) {
|
|
28
|
-
console.error(`Error: File not found: ${csvFilePath}`);
|
|
29
|
-
process.exit(1);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
const urlToRows = new Map(); // url -> [{ id, memberId }, ...]
|
|
33
|
-
let totalRows = 0;
|
|
34
|
-
let headersValidated = false;
|
|
35
|
-
let headers = null;
|
|
36
|
-
let urlColumnName = null;
|
|
37
|
-
let idColumnName = null;
|
|
38
|
-
let memberIdColumnName = null;
|
|
39
|
-
|
|
40
|
-
return new Promise((resolve, reject) => {
|
|
41
|
-
fs.createReadStream(csvFilePath)
|
|
42
|
-
.pipe(csv())
|
|
43
|
-
.on('headers', receivedHeaders => {
|
|
44
|
-
headers = receivedHeaders;
|
|
45
|
-
const normalizedHeaders = headers.map(h => {
|
|
46
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
47
|
-
return normalized.toLowerCase().trim();
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
const urlIndex = normalizedHeaders.indexOf('url');
|
|
51
|
-
const idIndex = normalizedHeaders.indexOf('id');
|
|
52
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
53
|
-
|
|
54
|
-
if (urlIndex === -1 || idIndex === -1 || memberIdIndex === -1) {
|
|
55
|
-
console.error(
|
|
56
|
-
'Error: CSV must contain "url", "ID", and "memberId" columns (case-insensitive)'
|
|
57
|
-
);
|
|
58
|
-
console.error(`Found columns: ${headers.join(', ')}`);
|
|
59
|
-
process.exit(1);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
urlColumnName = headers[urlIndex];
|
|
63
|
-
idColumnName = headers[idIndex];
|
|
64
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
65
|
-
headersValidated = true;
|
|
66
|
-
})
|
|
67
|
-
.on('data', row => {
|
|
68
|
-
if (!headersValidated) {
|
|
69
|
-
headers = Object.keys(row);
|
|
70
|
-
const normalizedHeaders = headers.map(h => {
|
|
71
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
72
|
-
return normalized.toLowerCase().trim();
|
|
73
|
-
});
|
|
74
|
-
const urlIndex = normalizedHeaders.indexOf('url');
|
|
75
|
-
const idIndex = normalizedHeaders.indexOf('id');
|
|
76
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
77
|
-
if (urlIndex === -1 || idIndex === -1 || memberIdIndex === -1) {
|
|
78
|
-
console.error(
|
|
79
|
-
'Error: CSV must contain "url", "ID", and "memberId" columns (case-insensitive)'
|
|
80
|
-
);
|
|
81
|
-
process.exit(1);
|
|
82
|
-
}
|
|
83
|
-
urlColumnName = headers[urlIndex];
|
|
84
|
-
idColumnName = headers[idIndex];
|
|
85
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
86
|
-
headersValidated = true;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
totalRows++;
|
|
90
|
-
|
|
91
|
-
const url = row[urlColumnName];
|
|
92
|
-
const id = row[idColumnName];
|
|
93
|
-
const memberId = row[memberIdColumnName];
|
|
94
|
-
|
|
95
|
-
if (!url || !id) {
|
|
96
|
-
return;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
const trimmedUrl = url.trim();
|
|
100
|
-
const trimmedId = id.trim();
|
|
101
|
-
const trimmedMemberId =
|
|
102
|
-
memberId != null && String(memberId).trim() !== '' ? String(memberId).trim() : null;
|
|
103
|
-
|
|
104
|
-
if (!urlToRows.has(trimmedUrl)) {
|
|
105
|
-
urlToRows.set(trimmedUrl, []);
|
|
106
|
-
}
|
|
107
|
-
urlToRows.get(trimmedUrl).push({ id: trimmedId, memberId: trimmedMemberId });
|
|
108
|
-
})
|
|
109
|
-
.on('error', error => {
|
|
110
|
-
console.error('Error reading CSV file:', error.message);
|
|
111
|
-
reject(error);
|
|
112
|
-
})
|
|
113
|
-
.on('end', () => {
|
|
114
|
-
if (!headersValidated) {
|
|
115
|
-
console.error('Error: Could not read CSV headers');
|
|
116
|
-
process.exit(1);
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
const groups = [];
|
|
120
|
-
for (const [url, rows] of urlToRows.entries()) {
|
|
121
|
-
if (rows.length > 1) {
|
|
122
|
-
const ids = rows.map(r => r.id);
|
|
123
|
-
const memberIds = rows.map(r => r.memberId).filter(Boolean);
|
|
124
|
-
groups.push({ url, ids, memberIds });
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
groups.sort((a, b) => {
|
|
129
|
-
if (b.ids.length !== a.ids.length) return b.ids.length - a.ids.length;
|
|
130
|
-
return a.url.localeCompare(b.url);
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
const totalIdsInGroups = groups.reduce((sum, g) => sum + g.ids.length, 0);
|
|
134
|
-
const duplicateUrls = groups.map(g => g.url);
|
|
135
|
-
const allMemberIdOccurrences = groups.flatMap(g => g.memberIds);
|
|
136
|
-
const memberIdsWithDuplicateUrls = [...new Set(allMemberIdOccurrences)];
|
|
137
|
-
|
|
138
|
-
const memberIdCounts = new Map();
|
|
139
|
-
for (const memberId of allMemberIdOccurrences) {
|
|
140
|
-
memberIdCounts.set(memberId, (memberIdCounts.get(memberId) || 0) + 1);
|
|
141
|
-
}
|
|
142
|
-
const nonUniqueMemberIds = [...memberIdCounts.entries()]
|
|
143
|
-
.filter(([, count]) => count > 1)
|
|
144
|
-
.map(([memberId, count]) => ({ memberId, count }))
|
|
145
|
-
.sort((a, b) => b.count - a.count);
|
|
146
|
-
|
|
147
|
-
const report = {
|
|
148
|
-
totalRowsProcessed: totalRows,
|
|
149
|
-
totalDuplicateGroups: groups.length,
|
|
150
|
-
totalIdsInDuplicateGroups: totalIdsInGroups,
|
|
151
|
-
totalMemberIdOccurrencesInDuplicateGroups: allMemberIdOccurrences.length,
|
|
152
|
-
uniqueMemberIdsWithDuplicateUrls: memberIdsWithDuplicateUrls.length,
|
|
153
|
-
nonUniqueMemberIdsCount: nonUniqueMemberIds.length,
|
|
154
|
-
duplicateUrls,
|
|
155
|
-
memberIdsWithDuplicateUrls,
|
|
156
|
-
nonUniqueMemberIds,
|
|
157
|
-
groups,
|
|
158
|
-
};
|
|
159
|
-
|
|
160
|
-
if (outputJsonPath) {
|
|
161
|
-
fs.writeFileSync(outputJsonPath, JSON.stringify(report, null, 2), 'utf8');
|
|
162
|
-
console.log(`Report written to: ${outputJsonPath}`);
|
|
163
|
-
} else {
|
|
164
|
-
const csvDir = path.dirname(csvFilePath);
|
|
165
|
-
const csvBasename = path.basename(csvFilePath, path.extname(csvFilePath));
|
|
166
|
-
const defaultPath = path.join(csvDir, `${csvBasename}-duplicate-url-groups.json`);
|
|
167
|
-
fs.writeFileSync(defaultPath, JSON.stringify(report, null, 2), 'utf8');
|
|
168
|
-
console.log(`Report written to: ${defaultPath}`);
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
console.log('\n=== Non-unique URL groups (same URL → list of IDs, memberIds) ===');
|
|
172
|
-
console.log(`Total rows processed: ${totalRows}`);
|
|
173
|
-
console.log(`Number of duplicate URL groups: ${groups.length}`);
|
|
174
|
-
console.log(`Total IDs in those groups: ${totalIdsInGroups}`);
|
|
175
|
-
console.log(`Total memberId occurrences in those groups: ${allMemberIdOccurrences.length}`);
|
|
176
|
-
console.log(`Unique memberIds in those groups: ${memberIdsWithDuplicateUrls.length}`);
|
|
177
|
-
console.log(
|
|
178
|
-
`MemberIds that appear more than once (non-unique): ${nonUniqueMemberIds.length}`
|
|
179
|
-
);
|
|
180
|
-
if (duplicateUrls.length > 0) {
|
|
181
|
-
console.log('\nFirst 15 duplicate URLs:');
|
|
182
|
-
duplicateUrls.slice(0, 15).forEach((url, i) => {
|
|
183
|
-
console.log(` ${i + 1}. ${url}`);
|
|
184
|
-
});
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
resolve(report);
|
|
188
|
-
});
|
|
189
|
-
});
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
if (require.main === module) {
|
|
193
|
-
const csvFilePath = process.argv[2];
|
|
194
|
-
const outputJsonPath = process.argv[3];
|
|
195
|
-
extractDuplicateUrlGroups(csvFilePath, outputJsonPath).catch(error => {
|
|
196
|
-
console.error('Fatal error:', error.message);
|
|
197
|
-
process.exit(1);
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
module.exports = { extractDuplicateUrlGroups };
|
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
const csv = require('csv-parser');
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Reads a Members Data CSV and finds duplicate memberIds: any memberId (memberId column)
|
|
8
|
-
* that appears in more than one row. Outputs a JSON report with the list of
|
|
9
|
-
* duplicate memberIds and, per memberId, the rows (e.g. ID, url) where it appears.
|
|
10
|
-
*
|
|
11
|
-
* Usage: node dev-only-scripts/find-duplicate-ids.js <path-to-csv> [output-json-path]
|
|
12
|
-
*
|
|
13
|
-
* CSV must have a "memberId" column (case-insensitive). Optional: "ID", "url" for row details.
|
|
14
|
-
*/
|
|
15
|
-
function findDuplicateMemberIds(csvFilePath, outputJsonPath) {
|
|
16
|
-
if (!csvFilePath) {
|
|
17
|
-
console.error('Error: CSV file path is required');
|
|
18
|
-
console.error(
|
|
19
|
-
'Usage: node dev-only-scripts/find-duplicate-ids.js <path-to-csv> [output-json-path]'
|
|
20
|
-
);
|
|
21
|
-
process.exit(1);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
if (!fs.existsSync(csvFilePath)) {
|
|
25
|
-
console.error(`Error: File not found: ${csvFilePath}`);
|
|
26
|
-
process.exit(1);
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
const memberIdToRows = new Map(); // memberId -> [{ rowNumber, id?, url? }, ...]
|
|
30
|
-
let totalRows = 0;
|
|
31
|
-
let headersValidated = false;
|
|
32
|
-
let headers = null;
|
|
33
|
-
let memberIdColumnName = null;
|
|
34
|
-
let idColumnName = null;
|
|
35
|
-
let urlColumnName = null;
|
|
36
|
-
|
|
37
|
-
return new Promise((resolve, reject) => {
|
|
38
|
-
fs.createReadStream(csvFilePath)
|
|
39
|
-
.pipe(csv())
|
|
40
|
-
.on('headers', receivedHeaders => {
|
|
41
|
-
headers = receivedHeaders;
|
|
42
|
-
const normalizedHeaders = headers.map(h => {
|
|
43
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
44
|
-
return normalized.toLowerCase().trim();
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
48
|
-
if (memberIdIndex === -1) {
|
|
49
|
-
console.error('Error: CSV must contain a "memberId" column (case-insensitive)');
|
|
50
|
-
console.error(`Found columns: ${headers.join(', ')}`);
|
|
51
|
-
process.exit(1);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
55
|
-
idColumnName = headers[normalizedHeaders.indexOf('id')] || null;
|
|
56
|
-
urlColumnName = headers[normalizedHeaders.indexOf('url')] || null;
|
|
57
|
-
headersValidated = true;
|
|
58
|
-
})
|
|
59
|
-
.on('data', row => {
|
|
60
|
-
if (!headersValidated) {
|
|
61
|
-
headers = Object.keys(row);
|
|
62
|
-
const normalizedHeaders = headers.map(h => {
|
|
63
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
64
|
-
return normalized.toLowerCase().trim();
|
|
65
|
-
});
|
|
66
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
67
|
-
if (memberIdIndex === -1) {
|
|
68
|
-
console.error('Error: CSV must contain a "memberId" column (case-insensitive)');
|
|
69
|
-
process.exit(1);
|
|
70
|
-
}
|
|
71
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
72
|
-
idColumnName = headers[normalizedHeaders.indexOf('id')] || null;
|
|
73
|
-
urlColumnName = headers[normalizedHeaders.indexOf('url')] || null;
|
|
74
|
-
headersValidated = true;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
totalRows++;
|
|
78
|
-
const memberId = row[memberIdColumnName];
|
|
79
|
-
if (memberId == null || String(memberId).trim() === '') return;
|
|
80
|
-
|
|
81
|
-
const trimmedMemberId = String(memberId).trim();
|
|
82
|
-
const rowInfo = { rowNumber: totalRows };
|
|
83
|
-
if (idColumnName && row[idColumnName] != null)
|
|
84
|
-
rowInfo.id = String(row[idColumnName]).trim();
|
|
85
|
-
if (urlColumnName && row[urlColumnName] != null)
|
|
86
|
-
rowInfo.url = String(row[urlColumnName]).trim();
|
|
87
|
-
|
|
88
|
-
if (!memberIdToRows.has(trimmedMemberId)) {
|
|
89
|
-
memberIdToRows.set(trimmedMemberId, []);
|
|
90
|
-
}
|
|
91
|
-
memberIdToRows.get(trimmedMemberId).push(rowInfo);
|
|
92
|
-
})
|
|
93
|
-
.on('error', error => {
|
|
94
|
-
console.error('Error reading CSV file:', error.message);
|
|
95
|
-
reject(error);
|
|
96
|
-
})
|
|
97
|
-
.on('end', () => {
|
|
98
|
-
if (!headersValidated) {
|
|
99
|
-
console.error('Error: Could not read CSV headers');
|
|
100
|
-
process.exit(1);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
const groups = [];
|
|
104
|
-
for (const [memberId, rows] of memberIdToRows.entries()) {
|
|
105
|
-
if (rows.length > 1) {
|
|
106
|
-
groups.push({ memberId, count: rows.length, rows });
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
groups.sort((a, b) => b.count - a.count);
|
|
111
|
-
|
|
112
|
-
const duplicateMemberIds = groups.map(g => g.memberId);
|
|
113
|
-
const totalDuplicateRows = groups.reduce((sum, g) => sum + g.count, 0);
|
|
114
|
-
|
|
115
|
-
const report = {
|
|
116
|
-
totalRowsProcessed: totalRows,
|
|
117
|
-
totalDuplicateMemberIds: groups.length,
|
|
118
|
-
totalRowsWithDuplicateMemberIds: totalDuplicateRows,
|
|
119
|
-
duplicateMemberIds,
|
|
120
|
-
groups,
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
if (outputJsonPath) {
|
|
124
|
-
fs.writeFileSync(outputJsonPath, JSON.stringify(report, null, 2), 'utf8');
|
|
125
|
-
console.log(`Report written to: ${outputJsonPath}`);
|
|
126
|
-
} else {
|
|
127
|
-
const csvDir = path.dirname(csvFilePath);
|
|
128
|
-
const csvBasename = path.basename(csvFilePath, path.extname(csvFilePath));
|
|
129
|
-
const defaultPath = path.join(csvDir, `${csvBasename}-duplicate-member-ids-report.json`);
|
|
130
|
-
fs.writeFileSync(defaultPath, JSON.stringify(report, null, 2), 'utf8');
|
|
131
|
-
console.log(`Report written to: ${defaultPath}`);
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
console.log('\n=== Duplicate memberIds Report ===');
|
|
135
|
-
console.log(`Total rows processed: ${totalRows}`);
|
|
136
|
-
console.log(`memberIds that appear more than once: ${groups.length}`);
|
|
137
|
-
console.log(`Total rows with those memberIds: ${totalDuplicateRows}`);
|
|
138
|
-
if (groups.length > 0) {
|
|
139
|
-
console.log('\nDuplicate memberIds (first 20):');
|
|
140
|
-
groups.slice(0, 20).forEach((g, i) => {
|
|
141
|
-
console.log(` ${i + 1}. ${g.memberId} (${g.count} rows)`);
|
|
142
|
-
});
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
resolve(report);
|
|
146
|
-
});
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
if (require.main === module) {
|
|
151
|
-
const csvFilePath = process.argv[2];
|
|
152
|
-
const outputJsonPath = process.argv[3];
|
|
153
|
-
findDuplicateMemberIds(csvFilePath, outputJsonPath).catch(error => {
|
|
154
|
-
console.error('Fatal error:', error.message);
|
|
155
|
-
process.exit(1);
|
|
156
|
-
});
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
module.exports = { findDuplicateMemberIds };
|