abmp-npm 2.0.76 → 2.0.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -1
- package/pages/Home.js +27 -55
- package/pages/index.js +0 -1
- package/public/Utils/personalDetailsUtils.js +5 -1
- package/public/Utils/sharedUtils.js +32 -0
- package/public/consts.js +0 -1
- package/.claude/worktrees/fix-save-button-disable/.husky/pre-commit +0 -19
- package/.claude/worktrees/fix-save-button-disable/.prettierignore +0 -7
- package/.claude/worktrees/fix-save-button-disable/.prettierrc.json +0 -16
- package/.claude/worktrees/fix-save-button-disable/README.md +0 -1
- package/.claude/worktrees/fix-save-button-disable/backend/README-member-contact-flows.md +0 -136
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/daily-pull-execution-check.test.js +0 -124
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/ensure-unique-urls-in-batch.test.js +0 -129
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/login-email-sync.test.js +0 -49
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/transient-retry-and-bulk-lookup.test.js +0 -129
- package/.claude/worktrees/fix-save-button-disable/backend/__tests__/url-uniqueness.test.js +0 -194
- package/.claude/worktrees/fix-save-button-disable/backend/automations-methods.js +0 -13
- package/.claude/worktrees/fix-save-button-disable/backend/cms-data-methods.js +0 -251
- package/.claude/worktrees/fix-save-button-disable/backend/consts.js +0 -58
- package/.claude/worktrees/fix-save-button-disable/backend/contacts-methods.js +0 -67
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/bulk-process-methods.js +0 -222
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/consts.js +0 -39
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/index.js +0 -5
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/process-member-methods.js +0 -213
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/schedule-methods.js +0 -41
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/sync-to-cms-methods.js +0 -161
- package/.claude/worktrees/fix-save-button-disable/backend/daily-pull/utils.js +0 -151
- package/.claude/worktrees/fix-save-button-disable/backend/data-hooks.js +0 -29
- package/.claude/worktrees/fix-save-button-disable/backend/dev-only-methods.js +0 -58
- package/.claude/worktrees/fix-save-button-disable/backend/elevated-modules.js +0 -18
- package/.claude/worktrees/fix-save-button-disable/backend/forms-methods.js +0 -52
- package/.claude/worktrees/fix-save-button-disable/backend/http-functions/httpFunctions.js +0 -86
- package/.claude/worktrees/fix-save-button-disable/backend/http-functions/index.js +0 -3
- package/.claude/worktrees/fix-save-button-disable/backend/http-functions/interests.js +0 -35
- package/.claude/worktrees/fix-save-button-disable/backend/index.js +0 -19
- package/.claude/worktrees/fix-save-button-disable/backend/jobs.js +0 -158
- package/.claude/worktrees/fix-save-button-disable/backend/login/generate-member-session-token.js +0 -27
- package/.claude/worktrees/fix-save-button-disable/backend/login/index.js +0 -8
- package/.claude/worktrees/fix-save-button-disable/backend/login/qa-login-methods.js +0 -54
- package/.claude/worktrees/fix-save-button-disable/backend/login/sso-methods.js +0 -159
- package/.claude/worktrees/fix-save-button-disable/backend/member-contact-orchestration.js +0 -111
- package/.claude/worktrees/fix-save-button-disable/backend/members-area-methods.js +0 -88
- package/.claude/worktrees/fix-save-button-disable/backend/members-data-methods.js +0 -789
- package/.claude/worktrees/fix-save-button-disable/backend/pac-api-methods.js +0 -54
- package/.claude/worktrees/fix-save-button-disable/backend/routers/index.js +0 -3
- package/.claude/worktrees/fix-save-button-disable/backend/routers/methods.js +0 -191
- package/.claude/worktrees/fix-save-button-disable/backend/routers/utils.js +0 -128
- package/.claude/worktrees/fix-save-button-disable/backend/search-filters-methods.js +0 -124
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/address-primary-methods.js +0 -256
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/address-visibility-methods.js +0 -160
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/consts.js +0 -34
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/daily-pull-check-methods.js +0 -78
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/email-normalize-methods.js +0 -134
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/index.js +0 -10
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/migration-methods.js +0 -46
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/tasks-configs.js +0 -251
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/tasks-helpers-methods.js +0 -433
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/tasks-process-methods.js +0 -606
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/url-migration-methods.js +0 -378
- package/.claude/worktrees/fix-save-button-disable/backend/tasks/url-space-fix-methods.js +0 -174
- package/.claude/worktrees/fix-save-button-disable/backend/utils.js +0 -277
- package/.claude/worktrees/fix-save-button-disable/dev-only-scripts/extract-duplicate-url-groups.js +0 -201
- package/.claude/worktrees/fix-save-button-disable/dev-only-scripts/find-duplicate-ids.js +0 -159
- package/.claude/worktrees/fix-save-button-disable/dev-only-scripts/find-duplicate-urls.js +0 -201
- package/.claude/worktrees/fix-save-button-disable/eslint.config.js +0 -120
- package/.claude/worktrees/fix-save-button-disable/index.js +0 -5
- package/.claude/worktrees/fix-save-button-disable/package-lock.json +0 -12160
- package/.claude/worktrees/fix-save-button-disable/package.json +0 -60
- package/.claude/worktrees/fix-save-button-disable/pages/ContactUs.js +0 -129
- package/.claude/worktrees/fix-save-button-disable/pages/Home.js +0 -779
- package/.claude/worktrees/fix-save-button-disable/pages/LearnMore.js +0 -27
- package/.claude/worktrees/fix-save-button-disable/pages/LoadingPage.js +0 -29
- package/.claude/worktrees/fix-save-button-disable/pages/Profile.js +0 -354
- package/.claude/worktrees/fix-save-button-disable/pages/QAPage.js +0 -39
- package/.claude/worktrees/fix-save-button-disable/pages/SaveAlerts.js +0 -13
- package/.claude/worktrees/fix-save-button-disable/pages/SelectBannerImages.js +0 -46
- package/.claude/worktrees/fix-save-button-disable/pages/deleteConfirm.js +0 -19
- package/.claude/worktrees/fix-save-button-disable/pages/index.js +0 -12
- package/.claude/worktrees/fix-save-button-disable/pages/personalDetails.js +0 -2486
- package/.claude/worktrees/fix-save-button-disable/public/Utils/homePage.js +0 -798
- package/.claude/worktrees/fix-save-button-disable/public/Utils/personalDetailsUtils.js +0 -25
- package/.claude/worktrees/fix-save-button-disable/public/Utils/sharedUtils.js +0 -235
- package/.claude/worktrees/fix-save-button-disable/public/consts.js +0 -113
- package/.claude/worktrees/fix-save-button-disable/public/index.js +0 -5
- package/.claude/worktrees/fix-save-button-disable/public/messages.js +0 -16
- package/.claude/worktrees/fix-save-button-disable/public/sso-auth-methods.js +0 -43
- package/.husky/pre-commit +0 -19
- package/.prettierignore +0 -7
- package/.prettierrc.json +0 -16
- package/backend/__tests__/daily-pull-execution-check.test.js +0 -124
- package/backend/__tests__/ensure-unique-urls-in-batch.test.js +0 -129
- package/backend/__tests__/login-email-sync.test.js +0 -49
- package/backend/__tests__/transient-retry-and-bulk-lookup.test.js +0 -129
- package/backend/__tests__/url-uniqueness.test.js +0 -194
- package/dev-only-scripts/extract-duplicate-url-groups.js +0 -201
- package/dev-only-scripts/find-duplicate-ids.js +0 -159
- package/dev-only-scripts/find-duplicate-urls.js +0 -201
- package/eslint.config.js +0 -120
- package/pages/contactForLocation.js +0 -28
|
@@ -1,201 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
const csv = require('csv-parser');
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Reads a Members Data CSV and extracts groups of non-unique URLs:
|
|
8
|
-
* each group is a URL plus the list of IDs and memberIds that share that URL.
|
|
9
|
-
* Only outputs groups where the same URL appears more than once.
|
|
10
|
-
* Report includes duplicateUrls, memberIdsWithDuplicateUrls (flat array of unique memberIds
|
|
11
|
-
* in duplicate groups), and nonUniqueMemberIds (memberIds that appear more than once, with count).
|
|
12
|
-
*
|
|
13
|
-
* Usage: node dev-only-scripts/extract-duplicate-url-groups.js <path-to-csv> [output-json-path]
|
|
14
|
-
* Example: node dev-only-scripts/extract-duplicate-url-groups.js "/Users/Besan/Downloads/Members+Data+Latest (17).csv"
|
|
15
|
-
*
|
|
16
|
-
* CSV must have columns: "url", "ID", and "memberId" (case-insensitive).
|
|
17
|
-
*/
|
|
18
|
-
function extractDuplicateUrlGroups(csvFilePath, outputJsonPath) {
|
|
19
|
-
if (!csvFilePath) {
|
|
20
|
-
console.error('Error: CSV file path is required');
|
|
21
|
-
console.error(
|
|
22
|
-
'Usage: node dev-only-scripts/extract-duplicate-url-groups.js <path-to-csv> [output-json-path]'
|
|
23
|
-
);
|
|
24
|
-
process.exit(1);
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
if (!fs.existsSync(csvFilePath)) {
|
|
28
|
-
console.error(`Error: File not found: ${csvFilePath}`);
|
|
29
|
-
process.exit(1);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
const urlToRows = new Map(); // url -> [{ id, memberId }, ...]
|
|
33
|
-
let totalRows = 0;
|
|
34
|
-
let headersValidated = false;
|
|
35
|
-
let headers = null;
|
|
36
|
-
let urlColumnName = null;
|
|
37
|
-
let idColumnName = null;
|
|
38
|
-
let memberIdColumnName = null;
|
|
39
|
-
|
|
40
|
-
return new Promise((resolve, reject) => {
|
|
41
|
-
fs.createReadStream(csvFilePath)
|
|
42
|
-
.pipe(csv())
|
|
43
|
-
.on('headers', receivedHeaders => {
|
|
44
|
-
headers = receivedHeaders;
|
|
45
|
-
const normalizedHeaders = headers.map(h => {
|
|
46
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
47
|
-
return normalized.toLowerCase().trim();
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
const urlIndex = normalizedHeaders.indexOf('url');
|
|
51
|
-
const idIndex = normalizedHeaders.indexOf('id');
|
|
52
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
53
|
-
|
|
54
|
-
if (urlIndex === -1 || idIndex === -1 || memberIdIndex === -1) {
|
|
55
|
-
console.error(
|
|
56
|
-
'Error: CSV must contain "url", "ID", and "memberId" columns (case-insensitive)'
|
|
57
|
-
);
|
|
58
|
-
console.error(`Found columns: ${headers.join(', ')}`);
|
|
59
|
-
process.exit(1);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
urlColumnName = headers[urlIndex];
|
|
63
|
-
idColumnName = headers[idIndex];
|
|
64
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
65
|
-
headersValidated = true;
|
|
66
|
-
})
|
|
67
|
-
.on('data', row => {
|
|
68
|
-
if (!headersValidated) {
|
|
69
|
-
headers = Object.keys(row);
|
|
70
|
-
const normalizedHeaders = headers.map(h => {
|
|
71
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
72
|
-
return normalized.toLowerCase().trim();
|
|
73
|
-
});
|
|
74
|
-
const urlIndex = normalizedHeaders.indexOf('url');
|
|
75
|
-
const idIndex = normalizedHeaders.indexOf('id');
|
|
76
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
77
|
-
if (urlIndex === -1 || idIndex === -1 || memberIdIndex === -1) {
|
|
78
|
-
console.error(
|
|
79
|
-
'Error: CSV must contain "url", "ID", and "memberId" columns (case-insensitive)'
|
|
80
|
-
);
|
|
81
|
-
process.exit(1);
|
|
82
|
-
}
|
|
83
|
-
urlColumnName = headers[urlIndex];
|
|
84
|
-
idColumnName = headers[idIndex];
|
|
85
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
86
|
-
headersValidated = true;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
totalRows++;
|
|
90
|
-
|
|
91
|
-
const url = row[urlColumnName];
|
|
92
|
-
const id = row[idColumnName];
|
|
93
|
-
const memberId = row[memberIdColumnName];
|
|
94
|
-
|
|
95
|
-
if (!url || !id) {
|
|
96
|
-
return;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
const trimmedUrl = url.trim();
|
|
100
|
-
const trimmedId = id.trim();
|
|
101
|
-
const trimmedMemberId =
|
|
102
|
-
memberId != null && String(memberId).trim() !== '' ? String(memberId).trim() : null;
|
|
103
|
-
|
|
104
|
-
if (!urlToRows.has(trimmedUrl)) {
|
|
105
|
-
urlToRows.set(trimmedUrl, []);
|
|
106
|
-
}
|
|
107
|
-
urlToRows.get(trimmedUrl).push({ id: trimmedId, memberId: trimmedMemberId });
|
|
108
|
-
})
|
|
109
|
-
.on('error', error => {
|
|
110
|
-
console.error('Error reading CSV file:', error.message);
|
|
111
|
-
reject(error);
|
|
112
|
-
})
|
|
113
|
-
.on('end', () => {
|
|
114
|
-
if (!headersValidated) {
|
|
115
|
-
console.error('Error: Could not read CSV headers');
|
|
116
|
-
process.exit(1);
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
const groups = [];
|
|
120
|
-
for (const [url, rows] of urlToRows.entries()) {
|
|
121
|
-
if (rows.length > 1) {
|
|
122
|
-
const ids = rows.map(r => r.id);
|
|
123
|
-
const memberIds = rows.map(r => r.memberId).filter(Boolean);
|
|
124
|
-
groups.push({ url, ids, memberIds });
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
groups.sort((a, b) => {
|
|
129
|
-
if (b.ids.length !== a.ids.length) return b.ids.length - a.ids.length;
|
|
130
|
-
return a.url.localeCompare(b.url);
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
const totalIdsInGroups = groups.reduce((sum, g) => sum + g.ids.length, 0);
|
|
134
|
-
const duplicateUrls = groups.map(g => g.url);
|
|
135
|
-
const allMemberIdOccurrences = groups.flatMap(g => g.memberIds);
|
|
136
|
-
const memberIdsWithDuplicateUrls = [...new Set(allMemberIdOccurrences)];
|
|
137
|
-
|
|
138
|
-
const memberIdCounts = new Map();
|
|
139
|
-
for (const memberId of allMemberIdOccurrences) {
|
|
140
|
-
memberIdCounts.set(memberId, (memberIdCounts.get(memberId) || 0) + 1);
|
|
141
|
-
}
|
|
142
|
-
const nonUniqueMemberIds = [...memberIdCounts.entries()]
|
|
143
|
-
.filter(([, count]) => count > 1)
|
|
144
|
-
.map(([memberId, count]) => ({ memberId, count }))
|
|
145
|
-
.sort((a, b) => b.count - a.count);
|
|
146
|
-
|
|
147
|
-
const report = {
|
|
148
|
-
totalRowsProcessed: totalRows,
|
|
149
|
-
totalDuplicateGroups: groups.length,
|
|
150
|
-
totalIdsInDuplicateGroups: totalIdsInGroups,
|
|
151
|
-
totalMemberIdOccurrencesInDuplicateGroups: allMemberIdOccurrences.length,
|
|
152
|
-
uniqueMemberIdsWithDuplicateUrls: memberIdsWithDuplicateUrls.length,
|
|
153
|
-
nonUniqueMemberIdsCount: nonUniqueMemberIds.length,
|
|
154
|
-
duplicateUrls,
|
|
155
|
-
memberIdsWithDuplicateUrls,
|
|
156
|
-
nonUniqueMemberIds,
|
|
157
|
-
groups,
|
|
158
|
-
};
|
|
159
|
-
|
|
160
|
-
if (outputJsonPath) {
|
|
161
|
-
fs.writeFileSync(outputJsonPath, JSON.stringify(report, null, 2), 'utf8');
|
|
162
|
-
console.log(`Report written to: ${outputJsonPath}`);
|
|
163
|
-
} else {
|
|
164
|
-
const csvDir = path.dirname(csvFilePath);
|
|
165
|
-
const csvBasename = path.basename(csvFilePath, path.extname(csvFilePath));
|
|
166
|
-
const defaultPath = path.join(csvDir, `${csvBasename}-duplicate-url-groups.json`);
|
|
167
|
-
fs.writeFileSync(defaultPath, JSON.stringify(report, null, 2), 'utf8');
|
|
168
|
-
console.log(`Report written to: ${defaultPath}`);
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
console.log('\n=== Non-unique URL groups (same URL → list of IDs, memberIds) ===');
|
|
172
|
-
console.log(`Total rows processed: ${totalRows}`);
|
|
173
|
-
console.log(`Number of duplicate URL groups: ${groups.length}`);
|
|
174
|
-
console.log(`Total IDs in those groups: ${totalIdsInGroups}`);
|
|
175
|
-
console.log(`Total memberId occurrences in those groups: ${allMemberIdOccurrences.length}`);
|
|
176
|
-
console.log(`Unique memberIds in those groups: ${memberIdsWithDuplicateUrls.length}`);
|
|
177
|
-
console.log(
|
|
178
|
-
`MemberIds that appear more than once (non-unique): ${nonUniqueMemberIds.length}`
|
|
179
|
-
);
|
|
180
|
-
if (duplicateUrls.length > 0) {
|
|
181
|
-
console.log('\nFirst 15 duplicate URLs:');
|
|
182
|
-
duplicateUrls.slice(0, 15).forEach((url, i) => {
|
|
183
|
-
console.log(` ${i + 1}. ${url}`);
|
|
184
|
-
});
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
resolve(report);
|
|
188
|
-
});
|
|
189
|
-
});
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
if (require.main === module) {
|
|
193
|
-
const csvFilePath = process.argv[2];
|
|
194
|
-
const outputJsonPath = process.argv[3];
|
|
195
|
-
extractDuplicateUrlGroups(csvFilePath, outputJsonPath).catch(error => {
|
|
196
|
-
console.error('Fatal error:', error.message);
|
|
197
|
-
process.exit(1);
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
module.exports = { extractDuplicateUrlGroups };
|
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
const csv = require('csv-parser');
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Reads a Members Data CSV and finds duplicate memberIds: any memberId (memberId column)
|
|
8
|
-
* that appears in more than one row. Outputs a JSON report with the list of
|
|
9
|
-
* duplicate memberIds and, per memberId, the rows (e.g. ID, url) where it appears.
|
|
10
|
-
*
|
|
11
|
-
* Usage: node dev-only-scripts/find-duplicate-ids.js <path-to-csv> [output-json-path]
|
|
12
|
-
*
|
|
13
|
-
* CSV must have a "memberId" column (case-insensitive). Optional: "ID", "url" for row details.
|
|
14
|
-
*/
|
|
15
|
-
function findDuplicateMemberIds(csvFilePath, outputJsonPath) {
|
|
16
|
-
if (!csvFilePath) {
|
|
17
|
-
console.error('Error: CSV file path is required');
|
|
18
|
-
console.error(
|
|
19
|
-
'Usage: node dev-only-scripts/find-duplicate-ids.js <path-to-csv> [output-json-path]'
|
|
20
|
-
);
|
|
21
|
-
process.exit(1);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
if (!fs.existsSync(csvFilePath)) {
|
|
25
|
-
console.error(`Error: File not found: ${csvFilePath}`);
|
|
26
|
-
process.exit(1);
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
const memberIdToRows = new Map(); // memberId -> [{ rowNumber, id?, url? }, ...]
|
|
30
|
-
let totalRows = 0;
|
|
31
|
-
let headersValidated = false;
|
|
32
|
-
let headers = null;
|
|
33
|
-
let memberIdColumnName = null;
|
|
34
|
-
let idColumnName = null;
|
|
35
|
-
let urlColumnName = null;
|
|
36
|
-
|
|
37
|
-
return new Promise((resolve, reject) => {
|
|
38
|
-
fs.createReadStream(csvFilePath)
|
|
39
|
-
.pipe(csv())
|
|
40
|
-
.on('headers', receivedHeaders => {
|
|
41
|
-
headers = receivedHeaders;
|
|
42
|
-
const normalizedHeaders = headers.map(h => {
|
|
43
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
44
|
-
return normalized.toLowerCase().trim();
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
48
|
-
if (memberIdIndex === -1) {
|
|
49
|
-
console.error('Error: CSV must contain a "memberId" column (case-insensitive)');
|
|
50
|
-
console.error(`Found columns: ${headers.join(', ')}`);
|
|
51
|
-
process.exit(1);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
55
|
-
idColumnName = headers[normalizedHeaders.indexOf('id')] || null;
|
|
56
|
-
urlColumnName = headers[normalizedHeaders.indexOf('url')] || null;
|
|
57
|
-
headersValidated = true;
|
|
58
|
-
})
|
|
59
|
-
.on('data', row => {
|
|
60
|
-
if (!headersValidated) {
|
|
61
|
-
headers = Object.keys(row);
|
|
62
|
-
const normalizedHeaders = headers.map(h => {
|
|
63
|
-
const normalized = String(h).trim().replace(/["']/g, '');
|
|
64
|
-
return normalized.toLowerCase().trim();
|
|
65
|
-
});
|
|
66
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
67
|
-
if (memberIdIndex === -1) {
|
|
68
|
-
console.error('Error: CSV must contain a "memberId" column (case-insensitive)');
|
|
69
|
-
process.exit(1);
|
|
70
|
-
}
|
|
71
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
72
|
-
idColumnName = headers[normalizedHeaders.indexOf('id')] || null;
|
|
73
|
-
urlColumnName = headers[normalizedHeaders.indexOf('url')] || null;
|
|
74
|
-
headersValidated = true;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
totalRows++;
|
|
78
|
-
const memberId = row[memberIdColumnName];
|
|
79
|
-
if (memberId == null || String(memberId).trim() === '') return;
|
|
80
|
-
|
|
81
|
-
const trimmedMemberId = String(memberId).trim();
|
|
82
|
-
const rowInfo = { rowNumber: totalRows };
|
|
83
|
-
if (idColumnName && row[idColumnName] != null)
|
|
84
|
-
rowInfo.id = String(row[idColumnName]).trim();
|
|
85
|
-
if (urlColumnName && row[urlColumnName] != null)
|
|
86
|
-
rowInfo.url = String(row[urlColumnName]).trim();
|
|
87
|
-
|
|
88
|
-
if (!memberIdToRows.has(trimmedMemberId)) {
|
|
89
|
-
memberIdToRows.set(trimmedMemberId, []);
|
|
90
|
-
}
|
|
91
|
-
memberIdToRows.get(trimmedMemberId).push(rowInfo);
|
|
92
|
-
})
|
|
93
|
-
.on('error', error => {
|
|
94
|
-
console.error('Error reading CSV file:', error.message);
|
|
95
|
-
reject(error);
|
|
96
|
-
})
|
|
97
|
-
.on('end', () => {
|
|
98
|
-
if (!headersValidated) {
|
|
99
|
-
console.error('Error: Could not read CSV headers');
|
|
100
|
-
process.exit(1);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
const groups = [];
|
|
104
|
-
for (const [memberId, rows] of memberIdToRows.entries()) {
|
|
105
|
-
if (rows.length > 1) {
|
|
106
|
-
groups.push({ memberId, count: rows.length, rows });
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
groups.sort((a, b) => b.count - a.count);
|
|
111
|
-
|
|
112
|
-
const duplicateMemberIds = groups.map(g => g.memberId);
|
|
113
|
-
const totalDuplicateRows = groups.reduce((sum, g) => sum + g.count, 0);
|
|
114
|
-
|
|
115
|
-
const report = {
|
|
116
|
-
totalRowsProcessed: totalRows,
|
|
117
|
-
totalDuplicateMemberIds: groups.length,
|
|
118
|
-
totalRowsWithDuplicateMemberIds: totalDuplicateRows,
|
|
119
|
-
duplicateMemberIds,
|
|
120
|
-
groups,
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
if (outputJsonPath) {
|
|
124
|
-
fs.writeFileSync(outputJsonPath, JSON.stringify(report, null, 2), 'utf8');
|
|
125
|
-
console.log(`Report written to: ${outputJsonPath}`);
|
|
126
|
-
} else {
|
|
127
|
-
const csvDir = path.dirname(csvFilePath);
|
|
128
|
-
const csvBasename = path.basename(csvFilePath, path.extname(csvFilePath));
|
|
129
|
-
const defaultPath = path.join(csvDir, `${csvBasename}-duplicate-member-ids-report.json`);
|
|
130
|
-
fs.writeFileSync(defaultPath, JSON.stringify(report, null, 2), 'utf8');
|
|
131
|
-
console.log(`Report written to: ${defaultPath}`);
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
console.log('\n=== Duplicate memberIds Report ===');
|
|
135
|
-
console.log(`Total rows processed: ${totalRows}`);
|
|
136
|
-
console.log(`memberIds that appear more than once: ${groups.length}`);
|
|
137
|
-
console.log(`Total rows with those memberIds: ${totalDuplicateRows}`);
|
|
138
|
-
if (groups.length > 0) {
|
|
139
|
-
console.log('\nDuplicate memberIds (first 20):');
|
|
140
|
-
groups.slice(0, 20).forEach((g, i) => {
|
|
141
|
-
console.log(` ${i + 1}. ${g.memberId} (${g.count} rows)`);
|
|
142
|
-
});
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
resolve(report);
|
|
146
|
-
});
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
if (require.main === module) {
|
|
151
|
-
const csvFilePath = process.argv[2];
|
|
152
|
-
const outputJsonPath = process.argv[3];
|
|
153
|
-
findDuplicateMemberIds(csvFilePath, outputJsonPath).catch(error => {
|
|
154
|
-
console.error('Fatal error:', error.message);
|
|
155
|
-
process.exit(1);
|
|
156
|
-
});
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
module.exports = { findDuplicateMemberIds };
|
|
@@ -1,201 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
const csv = require('csv-parser');
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Finds duplicate URLs in a CSV file and generates a JSON report
|
|
8
|
-
* Usage: node scripts/find-duplicate-urls.js <path-to-csv-file>
|
|
9
|
-
*/
|
|
10
|
-
function findDuplicateUrls(csvFilePath) {
|
|
11
|
-
// Validate command-line argument
|
|
12
|
-
if (!csvFilePath) {
|
|
13
|
-
console.error('Error: CSV file path is required');
|
|
14
|
-
console.error('Usage: node scripts/find-duplicate-urls.js <path-to-csv-file>');
|
|
15
|
-
process.exit(1);
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
// Validate file exists and is readable
|
|
19
|
-
if (!fs.existsSync(csvFilePath)) {
|
|
20
|
-
console.error(`Error: File not found: ${csvFilePath}`);
|
|
21
|
-
process.exit(1);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
const urlMap = new Map(); // url -> [memberId1, memberId2, ...]
|
|
25
|
-
let totalMembers = 0;
|
|
26
|
-
let rowNumber = 0;
|
|
27
|
-
let headersValidated = false;
|
|
28
|
-
let headers = null;
|
|
29
|
-
let urlColumnName = null;
|
|
30
|
-
let memberIdColumnName = null;
|
|
31
|
-
|
|
32
|
-
return new Promise((resolve, reject) => {
|
|
33
|
-
fs.createReadStream(csvFilePath)
|
|
34
|
-
.pipe(csv())
|
|
35
|
-
.on('headers', receivedHeaders => {
|
|
36
|
-
headers = receivedHeaders;
|
|
37
|
-
// Validate required columns exist - normalize by removing quotes, trimming, and lowercasing
|
|
38
|
-
const normalizedHeaders = headers.map(h => {
|
|
39
|
-
let normalized = String(h).trim();
|
|
40
|
-
// Remove all quotes (single and double) from the string
|
|
41
|
-
normalized = normalized.replace(/["']/g, '');
|
|
42
|
-
return normalized.toLowerCase().trim();
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
// Find the actual column names for url and memberId
|
|
46
|
-
const urlIndex = normalizedHeaders.indexOf('url');
|
|
47
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
48
|
-
|
|
49
|
-
if (urlIndex === -1 || memberIdIndex === -1) {
|
|
50
|
-
console.error('Error: CSV must contain "url" and "memberId" columns (case-insensitive)');
|
|
51
|
-
console.error(`Found columns: ${headers.join(', ')}`);
|
|
52
|
-
console.error(`Normalized columns: ${normalizedHeaders.join(', ')}`);
|
|
53
|
-
process.exit(1);
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// Store the actual column names (with original casing/quotes)
|
|
57
|
-
urlColumnName = headers[urlIndex];
|
|
58
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
59
|
-
headersValidated = true;
|
|
60
|
-
})
|
|
61
|
-
.on('data', row => {
|
|
62
|
-
// Validate headers on first data row if headers event didn't fire
|
|
63
|
-
if (!headersValidated) {
|
|
64
|
-
headers = Object.keys(row);
|
|
65
|
-
// Normalize by removing quotes, trimming, and lowercasing
|
|
66
|
-
const normalizedHeaders = headers.map(h => {
|
|
67
|
-
let normalized = String(h).trim();
|
|
68
|
-
// Remove all quotes (single and double) from the string
|
|
69
|
-
normalized = normalized.replace(/["']/g, '');
|
|
70
|
-
return normalized.toLowerCase().trim();
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
const urlIndex = normalizedHeaders.indexOf('url');
|
|
74
|
-
const memberIdIndex = normalizedHeaders.indexOf('memberid');
|
|
75
|
-
|
|
76
|
-
if (urlIndex === -1 || memberIdIndex === -1) {
|
|
77
|
-
console.error(
|
|
78
|
-
'Error: CSV must contain "url" and "memberId" columns (case-insensitive)'
|
|
79
|
-
);
|
|
80
|
-
console.error(`Found columns: ${headers.join(', ')}`);
|
|
81
|
-
console.error(`Normalized columns: ${normalizedHeaders.join(', ')}`);
|
|
82
|
-
process.exit(1);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
// Store the actual column names
|
|
86
|
-
urlColumnName = headers[urlIndex];
|
|
87
|
-
memberIdColumnName = headers[memberIdIndex];
|
|
88
|
-
headersValidated = true;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
rowNumber++;
|
|
92
|
-
totalMembers++;
|
|
93
|
-
|
|
94
|
-
// Get URL and memberId using the actual column names from headers
|
|
95
|
-
const url = row[urlColumnName];
|
|
96
|
-
const memberId = row[memberIdColumnName];
|
|
97
|
-
|
|
98
|
-
// Skip rows with missing URL or memberId
|
|
99
|
-
if (!url || !memberId) {
|
|
100
|
-
console.warn(
|
|
101
|
-
`Warning: Row ${rowNumber} skipped - missing url or memberId (url: ${url}, memberId: ${memberId})`
|
|
102
|
-
);
|
|
103
|
-
return;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
const trimmedUrl = url.trim();
|
|
107
|
-
const trimmedMemberId = memberId.trim();
|
|
108
|
-
|
|
109
|
-
// Track URL occurrences
|
|
110
|
-
if (!urlMap.has(trimmedUrl)) {
|
|
111
|
-
urlMap.set(trimmedUrl, []);
|
|
112
|
-
}
|
|
113
|
-
urlMap.get(trimmedUrl).push(trimmedMemberId);
|
|
114
|
-
})
|
|
115
|
-
.on('error', error => {
|
|
116
|
-
console.error('Error reading CSV file:', error.message);
|
|
117
|
-
reject(error);
|
|
118
|
-
})
|
|
119
|
-
.on('end', () => {
|
|
120
|
-
if (!headersValidated) {
|
|
121
|
-
console.error('Error: Could not read CSV headers');
|
|
122
|
-
process.exit(1);
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
// Find duplicates (URLs with count > 1)
|
|
126
|
-
const duplicateUrls = [];
|
|
127
|
-
let totalDuplicates = 0;
|
|
128
|
-
|
|
129
|
-
for (const [url, memberIds] of urlMap.entries()) {
|
|
130
|
-
if (memberIds.length > 1) {
|
|
131
|
-
duplicateUrls.push({
|
|
132
|
-
url: url,
|
|
133
|
-
count: memberIds.length,
|
|
134
|
-
memberIds: memberIds,
|
|
135
|
-
});
|
|
136
|
-
totalDuplicates += memberIds.length;
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
// Sort by count (descending) then by URL (ascending)
|
|
141
|
-
duplicateUrls.sort((a, b) => {
|
|
142
|
-
if (b.count !== a.count) {
|
|
143
|
-
return b.count - a.count;
|
|
144
|
-
}
|
|
145
|
-
return a.url.localeCompare(b.url);
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
const totalUniqueUrls = urlMap.size;
|
|
149
|
-
const uniqueDuplicateUrls = duplicateUrls.length;
|
|
150
|
-
|
|
151
|
-
// Create a simple list of duplicated URLs (just the URL strings)
|
|
152
|
-
const duplicatedUrlsList = duplicateUrls.map(item => item.url);
|
|
153
|
-
|
|
154
|
-
// Generate report
|
|
155
|
-
const report = {
|
|
156
|
-
totalMembers: totalMembers,
|
|
157
|
-
totalUniqueUrls: totalUniqueUrls,
|
|
158
|
-
duplicateUrls: duplicateUrls,
|
|
159
|
-
duplicatedUrlsList: duplicatedUrlsList,
|
|
160
|
-
summary: {
|
|
161
|
-
totalDuplicates: totalDuplicates,
|
|
162
|
-
uniqueDuplicateUrls: uniqueDuplicateUrls,
|
|
163
|
-
},
|
|
164
|
-
};
|
|
165
|
-
|
|
166
|
-
// Generate output filename
|
|
167
|
-
const csvDir = path.dirname(csvFilePath);
|
|
168
|
-
const csvBasename = path.basename(csvFilePath, path.extname(csvFilePath));
|
|
169
|
-
const outputPath = path.join(csvDir, `${csvBasename}-duplicate-urls-report.json`);
|
|
170
|
-
|
|
171
|
-
// Write JSON report
|
|
172
|
-
fs.writeFileSync(outputPath, JSON.stringify(report, null, 2), 'utf8');
|
|
173
|
-
|
|
174
|
-
console.log('\n=== Duplicate URL Report ===');
|
|
175
|
-
console.log(`Total members processed: ${totalMembers}`);
|
|
176
|
-
console.log(`Total unique URLs: ${totalUniqueUrls}`);
|
|
177
|
-
console.log(`Unique URLs with duplicates: ${uniqueDuplicateUrls}`);
|
|
178
|
-
console.log(`Total duplicate entries: ${totalDuplicates}`);
|
|
179
|
-
console.log(`\nReport saved to: ${outputPath}`);
|
|
180
|
-
console.log(`\nTop 10 most duplicated URLs:`);
|
|
181
|
-
duplicateUrls.slice(0, 10).forEach((item, index) => {
|
|
182
|
-
console.log(
|
|
183
|
-
` ${index + 1}. "${item.url}" - appears ${item.count} times (memberIds: ${item.memberIds.join(', ')})`
|
|
184
|
-
);
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
resolve(report);
|
|
188
|
-
});
|
|
189
|
-
});
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
// Run if executed directly
|
|
193
|
-
if (require.main === module) {
|
|
194
|
-
const csvFilePath = process.argv[2];
|
|
195
|
-
findDuplicateUrls(csvFilePath).catch(error => {
|
|
196
|
-
console.error('Fatal error:', error.message);
|
|
197
|
-
process.exit(1);
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
module.exports = { findDuplicateUrls };
|
package/eslint.config.js
DELETED
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
const js = require('@eslint/js');
|
|
2
|
-
const prettierConfig = require('eslint-config-prettier');
|
|
3
|
-
const importPlugin = require('eslint-plugin-import');
|
|
4
|
-
const prettier = require('eslint-plugin-prettier');
|
|
5
|
-
const promisePlugin = require('eslint-plugin-promise');
|
|
6
|
-
const globals = require('globals');
|
|
7
|
-
|
|
8
|
-
module.exports = [
|
|
9
|
-
// Recommended base configurations
|
|
10
|
-
js.configs.recommended,
|
|
11
|
-
prettierConfig,
|
|
12
|
-
|
|
13
|
-
// Main configuration
|
|
14
|
-
{
|
|
15
|
-
files: ['**/*.js'],
|
|
16
|
-
languageOptions: {
|
|
17
|
-
ecmaVersion: 2021,
|
|
18
|
-
sourceType: 'commonjs',
|
|
19
|
-
globals: {
|
|
20
|
-
...globals.node,
|
|
21
|
-
...globals.es2021,
|
|
22
|
-
...globals.jest,
|
|
23
|
-
},
|
|
24
|
-
},
|
|
25
|
-
plugins: {
|
|
26
|
-
prettier,
|
|
27
|
-
import: importPlugin,
|
|
28
|
-
promise: promisePlugin,
|
|
29
|
-
},
|
|
30
|
-
rules: {
|
|
31
|
-
// Error prevention
|
|
32
|
-
'no-var': 'error',
|
|
33
|
-
'no-unused-vars': ['error', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' }],
|
|
34
|
-
'no-console': ['warn', { allow: ['warn', 'error', 'log', 'info', 'group', 'groupEnd'] }],
|
|
35
|
-
'no-debugger': 'warn',
|
|
36
|
-
'no-duplicate-imports': 'error',
|
|
37
|
-
'no-unused-expressions': ['error', { allowShortCircuit: true, allowTernary: true }],
|
|
38
|
-
'no-use-before-define': ['error', { functions: false }],
|
|
39
|
-
|
|
40
|
-
// Best practices
|
|
41
|
-
'prefer-const': 'error',
|
|
42
|
-
'no-const-assign': 'error',
|
|
43
|
-
'prefer-arrow-callback': 'error',
|
|
44
|
-
'arrow-body-style': ['error', 'as-needed'],
|
|
45
|
-
'no-return-await': 'off',
|
|
46
|
-
'require-await': 'warn',
|
|
47
|
-
|
|
48
|
-
// Import rules
|
|
49
|
-
'import/order': [
|
|
50
|
-
'error',
|
|
51
|
-
{
|
|
52
|
-
groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
|
|
53
|
-
'newlines-between': 'always',
|
|
54
|
-
alphabetize: { order: 'asc', caseInsensitive: true },
|
|
55
|
-
},
|
|
56
|
-
],
|
|
57
|
-
'import/no-unresolved': [
|
|
58
|
-
'error',
|
|
59
|
-
{
|
|
60
|
-
commonjs: true,
|
|
61
|
-
caseSensitive: true,
|
|
62
|
-
},
|
|
63
|
-
],
|
|
64
|
-
'import/no-duplicates': 'error',
|
|
65
|
-
'import/no-commonjs': 'off',
|
|
66
|
-
'import/no-dynamic-require': 'off',
|
|
67
|
-
|
|
68
|
-
// Promise rules
|
|
69
|
-
'promise/always-return': 'off',
|
|
70
|
-
'promise/no-return-wrap': 'error',
|
|
71
|
-
'promise/param-names': 'error',
|
|
72
|
-
'promise/catch-or-return': 'error',
|
|
73
|
-
'promise/no-native': 'off',
|
|
74
|
-
'promise/no-callback-in-promise': 'warn',
|
|
75
|
-
'promise/no-promise-in-callback': 'warn',
|
|
76
|
-
'promise/no-nesting': 'warn',
|
|
77
|
-
|
|
78
|
-
// Prettier rules
|
|
79
|
-
'prettier/prettier': [
|
|
80
|
-
'error',
|
|
81
|
-
{
|
|
82
|
-
singleQuote: true,
|
|
83
|
-
trailingComma: 'es5',
|
|
84
|
-
printWidth: 100,
|
|
85
|
-
tabWidth: 2,
|
|
86
|
-
semi: true,
|
|
87
|
-
bracketSpacing: true,
|
|
88
|
-
arrowParens: 'avoid',
|
|
89
|
-
endOfLine: 'lf',
|
|
90
|
-
},
|
|
91
|
-
],
|
|
92
|
-
},
|
|
93
|
-
settings: {
|
|
94
|
-
'import/resolver': {
|
|
95
|
-
node: {
|
|
96
|
-
extensions: ['.js', '.jsx', '.ts', '.tsx'],
|
|
97
|
-
moduleDirectory: ['node_modules', 'public', 'backend', 'pages'],
|
|
98
|
-
},
|
|
99
|
-
},
|
|
100
|
-
},
|
|
101
|
-
},
|
|
102
|
-
|
|
103
|
-
// Test files override
|
|
104
|
-
{
|
|
105
|
-
files: ['**/*.test.js', '**/*.spec.js'],
|
|
106
|
-
languageOptions: {
|
|
107
|
-
globals: {
|
|
108
|
-
...globals.jest,
|
|
109
|
-
},
|
|
110
|
-
},
|
|
111
|
-
rules: {
|
|
112
|
-
'no-console': 'off',
|
|
113
|
-
},
|
|
114
|
-
},
|
|
115
|
-
|
|
116
|
-
// Ignore patterns
|
|
117
|
-
{
|
|
118
|
-
ignores: ['node_modules/**', 'dist/**', 'build/**', 'coverage/**', '*.min.js'],
|
|
119
|
-
},
|
|
120
|
-
];
|