codymaster 4.4.5 → 4.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +29 -14
  3. package/commands/demo.md +1 -1
  4. package/dist/context-bus.js +70 -0
  5. package/dist/context-db.js +265 -0
  6. package/dist/continuity.js +12 -0
  7. package/dist/file-watcher.js +79 -0
  8. package/dist/index.js +152 -1
  9. package/dist/l0-indexer.js +158 -0
  10. package/dist/mcp-context-server.js +400 -0
  11. package/dist/migrate-json-to-sqlite.js +126 -0
  12. package/dist/skill-chain.js +19 -3
  13. package/dist/token-budget.js +108 -0
  14. package/dist/uri-resolver.js +203 -0
  15. package/package.json +5 -1
  16. package/scripts/gate-0-secrets.js +63 -0
  17. package/scripts/gate-1-syntax.js +53 -0
  18. package/scripts/gate-5-dist-verify.js +55 -0
  19. package/scripts/gate-6-smoke-test.js +30 -0
  20. package/scripts/index-codebase.sh +552 -0
  21. package/scripts/mcp-bridge.js +284 -0
  22. package/scripts/postinstall.js +301 -0
  23. package/scripts/security-fixer.js +143 -0
  24. package/scripts/security-scan.js +55 -0
  25. package/scripts/test-gemini.js +13 -0
  26. package/scripts/todo-bridge.js +112 -0
  27. package/skills/_shared/helpers.md +50 -14
  28. package/skills/cm-autopilot/SKILL.md +29 -0
  29. package/skills/cm-autopilot/scripts/autopilot.py +190 -0
  30. package/skills/cm-continuity/SKILL.md +90 -28
  31. package/skills/cm-skill-chain/SKILL.md +47 -1
  32. package/skills/cm-start/SKILL.md +11 -2
  33. package/skills/boxme-git-config/SKILL.md +0 -56
  34. package/skills/boxme-local-dev/SKILL.md +0 -66
  35. package/skills/jobs-to-be-done/SKILL.md +0 -266
  36. package/skills/jobs-to-be-done/references/case-studies.md +0 -154
  37. package/skills/jobs-to-be-done/references/competitive-strategy.md +0 -280
  38. package/skills/jobs-to-be-done/references/diagnostics.md +0 -158
  39. package/skills/jobs-to-be-done/references/innovation-process.md +0 -392
  40. package/skills/jobs-to-be-done/references/organizational-change.md +0 -328
  41. package/skills/marketplace-report-crawler/SKILL.md +0 -176
  42. package/skills/marketplace-report-crawler/config/accounts.json +0 -41
  43. package/skills/marketplace-report-crawler/config/report-types.json +0 -422
  44. package/skills/marketplace-report-crawler/config/sessions.json +0 -3
  45. package/skills/marketplace-report-crawler/scripts/ab-wrapper.sh +0 -102
  46. package/skills/marketplace-report-crawler/scripts/browser-actions/lazada/lazada-actions.js +0 -114
  47. package/skills/marketplace-report-crawler/scripts/browser-actions/shopee/shopee-actions.js +0 -94
  48. package/skills/marketplace-report-crawler/scripts/browser-actions/tiktok/tiktok-actions.js +0 -272
  49. package/skills/marketplace-report-crawler/scripts/crawl-runner.js +0 -281
  50. package/skills/marketplace-report-crawler/scripts/session-check.sh +0 -72
  51. package/skills/marketplace-report-crawler/scripts/session-manager.sh +0 -349
  52. package/skills/marketplace-report-crawler/scripts/setup-folders.sh +0 -83
  53. package/skills/medical-research/SKILL.md +0 -194
  54. package/skills/medical-research/scripts/evidence_checker.py +0 -288
  55. package/skills/mom-test/SKILL.md +0 -267
  56. package/skills/mom-test/references/avoiding-bad-data.md +0 -221
  57. package/skills/mom-test/references/case-studies.md +0 -306
  58. package/skills/mom-test/references/commitment-advancement.md +0 -219
  59. package/skills/mom-test/references/finding-conversations.md +0 -251
  60. package/skills/mom-test/references/processing-learning.md +0 -256
  61. package/skills/mom-test/references/question-patterns.md +0 -198
  62. package/skills/pandasai-analytics/SKILL.md +0 -251
  63. package/skills/release-it/SKILL.md +0 -235
  64. package/skills/release-it/references/anti-patterns.md +0 -279
  65. package/skills/release-it/references/capacity-planning.md +0 -285
  66. package/skills/release-it/references/chaos-engineering.md +0 -325
  67. package/skills/release-it/references/deployment-strategies.md +0 -331
  68. package/skills/release-it/references/observability.md +0 -301
  69. package/skills/release-it/references/stability-patterns.md +0 -355
  70. package/skills/skill-creator-ultra/.agents/workflows/skill-audit.md +0 -37
  71. package/skills/skill-creator-ultra/.agents/workflows/skill-compare.md +0 -34
  72. package/skills/skill-creator-ultra/.agents/workflows/skill-export.md +0 -51
  73. package/skills/skill-creator-ultra/.agents/workflows/skill-generate.md +0 -39
  74. package/skills/skill-creator-ultra/.agents/workflows/skill-scaffold.md +0 -52
  75. package/skills/skill-creator-ultra/.agents/workflows/skill-simulate.md +0 -25
  76. package/skills/skill-creator-ultra/.agents/workflows/skill-stats.md +0 -31
  77. package/skills/skill-creator-ultra/.agents/workflows/skill-validate.md +0 -25
  78. package/skills/skill-creator-ultra/README.md +0 -1242
  79. package/skills/skill-creator-ultra/SKILL.md +0 -388
  80. package/skills/skill-creator-ultra/agents/analyzer.md +0 -274
  81. package/skills/skill-creator-ultra/agents/comparator.md +0 -202
  82. package/skills/skill-creator-ultra/agents/grader.md +0 -223
  83. package/skills/skill-creator-ultra/assets/eval_review.html +0 -146
  84. package/skills/skill-creator-ultra/eval-viewer/generate_review.py +0 -471
  85. package/skills/skill-creator-ultra/eval-viewer/viewer.html +0 -1325
  86. package/skills/skill-creator-ultra/examples/example_anthropic_frontend.md +0 -109
  87. package/skills/skill-creator-ultra/examples/example_anthropic_pdf.md +0 -116
  88. package/skills/skill-creator-ultra/examples/example_api_docs.md +0 -189
  89. package/skills/skill-creator-ultra/examples/example_db_migration.md +0 -253
  90. package/skills/skill-creator-ultra/examples/example_git_commit.md +0 -111
  91. package/skills/skill-creator-ultra/install.ps1 +0 -289
  92. package/skills/skill-creator-ultra/install.sh +0 -313
  93. package/skills/skill-creator-ultra/phases/phase1_interview.md +0 -202
  94. package/skills/skill-creator-ultra/phases/phase2_extract.md +0 -55
  95. package/skills/skill-creator-ultra/phases/phase3_detect.md +0 -57
  96. package/skills/skill-creator-ultra/phases/phase4_generate.md +0 -543
  97. package/skills/skill-creator-ultra/phases/phase5_test.md +0 -319
  98. package/skills/skill-creator-ultra/phases/phase6_eval.md +0 -301
  99. package/skills/skill-creator-ultra/phases/phase7_iterate.md +0 -103
  100. package/skills/skill-creator-ultra/phases/phase8_optimize.md +0 -113
  101. package/skills/skill-creator-ultra/resources/advanced_patterns.md +0 -499
  102. package/skills/skill-creator-ultra/resources/anti_patterns.md +0 -376
  103. package/skills/skill-creator-ultra/resources/blueprints.md +0 -498
  104. package/skills/skill-creator-ultra/resources/checklist.md +0 -243
  105. package/skills/skill-creator-ultra/resources/composition_cookbook.md +0 -291
  106. package/skills/skill-creator-ultra/resources/description_optimization.md +0 -90
  107. package/skills/skill-creator-ultra/resources/eval_guide.md +0 -133
  108. package/skills/skill-creator-ultra/resources/industry_questions.md +0 -189
  109. package/skills/skill-creator-ultra/resources/interview_questions.md +0 -200
  110. package/skills/skill-creator-ultra/resources/pattern_detection.md +0 -200
  111. package/skills/skill-creator-ultra/resources/prompt_engineering.md +0 -531
  112. package/skills/skill-creator-ultra/resources/schemas.md +0 -430
  113. package/skills/skill-creator-ultra/resources/script_integration.md +0 -593
  114. package/skills/skill-creator-ultra/resources/scripts_guide.md +0 -339
  115. package/skills/skill-creator-ultra/resources/skill_template.md +0 -124
  116. package/skills/skill-creator-ultra/resources/skill_writing_guide.md +0 -634
  117. package/skills/skill-creator-ultra/resources/versioning_guide.md +0 -193
  118. package/skills/skill-creator-ultra/scripts/ci_eval.py +0 -200
  119. package/skills/skill-creator-ultra/scripts/package_skill.py +0 -165
  120. package/skills/skill-creator-ultra/scripts/simulate_skill.py +0 -398
  121. package/skills/skill-creator-ultra/scripts/skill_audit.py +0 -611
  122. package/skills/skill-creator-ultra/scripts/skill_compare.py +0 -265
  123. package/skills/skill-creator-ultra/scripts/skill_export.py +0 -334
  124. package/skills/skill-creator-ultra/scripts/skill_scaffold.py +0 -403
  125. package/skills/skill-creator-ultra/scripts/skill_stats.py +0 -339
  126. package/skills/skill-creator-ultra/scripts/validate_skill.py +0 -411
  127. package/skills/tailwind-mastery/SKILL.md +0 -229
  128. package/skills/vercel-react-best-practices/AGENTS.md +0 -3373
  129. package/skills/vercel-react-best-practices/README.md +0 -123
  130. package/skills/vercel-react-best-practices/SKILL.md +0 -143
  131. package/skills/vercel-react-best-practices/rules/_sections.md +0 -46
  132. package/skills/vercel-react-best-practices/rules/_template.md +0 -28
  133. package/skills/vercel-react-best-practices/rules/advanced-event-handler-refs.md +0 -55
  134. package/skills/vercel-react-best-practices/rules/advanced-init-once.md +0 -42
  135. package/skills/vercel-react-best-practices/rules/advanced-use-latest.md +0 -39
  136. package/skills/vercel-react-best-practices/rules/async-api-routes.md +0 -38
  137. package/skills/vercel-react-best-practices/rules/async-defer-await.md +0 -80
  138. package/skills/vercel-react-best-practices/rules/async-dependencies.md +0 -51
  139. package/skills/vercel-react-best-practices/rules/async-parallel.md +0 -28
  140. package/skills/vercel-react-best-practices/rules/async-suspense-boundaries.md +0 -99
  141. package/skills/vercel-react-best-practices/rules/bundle-barrel-imports.md +0 -59
  142. package/skills/vercel-react-best-practices/rules/bundle-conditional.md +0 -31
  143. package/skills/vercel-react-best-practices/rules/bundle-defer-third-party.md +0 -49
  144. package/skills/vercel-react-best-practices/rules/bundle-dynamic-imports.md +0 -35
  145. package/skills/vercel-react-best-practices/rules/bundle-preload.md +0 -50
  146. package/skills/vercel-react-best-practices/rules/client-event-listeners.md +0 -74
  147. package/skills/vercel-react-best-practices/rules/client-localstorage-schema.md +0 -71
  148. package/skills/vercel-react-best-practices/rules/client-passive-event-listeners.md +0 -48
  149. package/skills/vercel-react-best-practices/rules/client-swr-dedup.md +0 -56
  150. package/skills/vercel-react-best-practices/rules/js-batch-dom-css.md +0 -107
  151. package/skills/vercel-react-best-practices/rules/js-cache-function-results.md +0 -80
  152. package/skills/vercel-react-best-practices/rules/js-cache-property-access.md +0 -28
  153. package/skills/vercel-react-best-practices/rules/js-cache-storage.md +0 -70
  154. package/skills/vercel-react-best-practices/rules/js-combine-iterations.md +0 -32
  155. package/skills/vercel-react-best-practices/rules/js-early-exit.md +0 -50
  156. package/skills/vercel-react-best-practices/rules/js-flatmap-filter.md +0 -60
  157. package/skills/vercel-react-best-practices/rules/js-hoist-regexp.md +0 -45
  158. package/skills/vercel-react-best-practices/rules/js-index-maps.md +0 -37
  159. package/skills/vercel-react-best-practices/rules/js-length-check-first.md +0 -49
  160. package/skills/vercel-react-best-practices/rules/js-min-max-loop.md +0 -82
  161. package/skills/vercel-react-best-practices/rules/js-set-map-lookups.md +0 -24
  162. package/skills/vercel-react-best-practices/rules/js-tosorted-immutable.md +0 -57
  163. package/skills/vercel-react-best-practices/rules/rendering-activity.md +0 -26
  164. package/skills/vercel-react-best-practices/rules/rendering-animate-svg-wrapper.md +0 -47
  165. package/skills/vercel-react-best-practices/rules/rendering-conditional-render.md +0 -40
  166. package/skills/vercel-react-best-practices/rules/rendering-content-visibility.md +0 -38
  167. package/skills/vercel-react-best-practices/rules/rendering-hoist-jsx.md +0 -46
  168. package/skills/vercel-react-best-practices/rules/rendering-hydration-no-flicker.md +0 -82
  169. package/skills/vercel-react-best-practices/rules/rendering-hydration-suppress-warning.md +0 -30
  170. package/skills/vercel-react-best-practices/rules/rendering-resource-hints.md +0 -85
  171. package/skills/vercel-react-best-practices/rules/rendering-script-defer-async.md +0 -68
  172. package/skills/vercel-react-best-practices/rules/rendering-svg-precision.md +0 -28
  173. package/skills/vercel-react-best-practices/rules/rendering-usetransition-loading.md +0 -75
  174. package/skills/vercel-react-best-practices/rules/rerender-defer-reads.md +0 -39
  175. package/skills/vercel-react-best-practices/rules/rerender-dependencies.md +0 -45
  176. package/skills/vercel-react-best-practices/rules/rerender-derived-state-no-effect.md +0 -40
  177. package/skills/vercel-react-best-practices/rules/rerender-derived-state.md +0 -29
  178. package/skills/vercel-react-best-practices/rules/rerender-functional-setstate.md +0 -74
  179. package/skills/vercel-react-best-practices/rules/rerender-lazy-state-init.md +0 -58
  180. package/skills/vercel-react-best-practices/rules/rerender-memo-with-default-value.md +0 -38
  181. package/skills/vercel-react-best-practices/rules/rerender-memo.md +0 -44
  182. package/skills/vercel-react-best-practices/rules/rerender-move-effect-to-event.md +0 -45
  183. package/skills/vercel-react-best-practices/rules/rerender-no-inline-components.md +0 -82
  184. package/skills/vercel-react-best-practices/rules/rerender-simple-expression-in-memo.md +0 -35
  185. package/skills/vercel-react-best-practices/rules/rerender-split-combined-hooks.md +0 -64
  186. package/skills/vercel-react-best-practices/rules/rerender-transitions.md +0 -40
  187. package/skills/vercel-react-best-practices/rules/rerender-use-deferred-value.md +0 -59
  188. package/skills/vercel-react-best-practices/rules/rerender-use-ref-transient-values.md +0 -73
  189. package/skills/vercel-react-best-practices/rules/server-after-nonblocking.md +0 -73
  190. package/skills/vercel-react-best-practices/rules/server-auth-actions.md +0 -96
  191. package/skills/vercel-react-best-practices/rules/server-cache-lru.md +0 -41
  192. package/skills/vercel-react-best-practices/rules/server-cache-react.md +0 -76
  193. package/skills/vercel-react-best-practices/rules/server-dedup-props.md +0 -65
  194. package/skills/vercel-react-best-practices/rules/server-hoist-static-io.md +0 -142
  195. package/skills/vercel-react-best-practices/rules/server-parallel-fetching.md +0 -83
  196. package/skills/vercel-react-best-practices/rules/server-serialization.md +0 -38
  197. package/skills/web-design-guidelines/SKILL.md +0 -39
@@ -1,281 +0,0 @@
1
- /**
2
- * Marketplace Report Crawl Runner
3
- * ================================
4
- * Orchestrator that reads config files and generates crawl instructions
5
- * for the browser agent to execute.
6
- *
7
- * UPDATED: Now reads from accounts.csv (via csv-reader.js) instead of accounts.json
8
- * Includes logging integration for activity & bug tracking.
9
- *
10
- * Usage:
11
- * node crawl-runner.js --platform shopee --brand "Vita Dairy" --period 202602
12
- * node crawl-runner.js --list # List all report types
13
- * node crawl-runner.js --setup-folders ./data # Generate folder structure
14
- * node crawl-runner.js --status ./data 202602 # Check download status
15
- * node crawl-runner.js --dashboard # Open dashboard
16
- */
17
-
18
- const fs = require('fs');
19
- const path = require('path');
20
-
21
- // --- Load configs ---
22
- const configDir = path.join(__dirname, '..', 'config');
23
- const { loadAccounts, getAccountsByBrand, getBrands } = require(
24
- path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'scripts', 'csv-reader')
25
- );
26
- const logger = require(
27
- path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'scripts', 'logger')
28
- );
29
-
30
- let accountsList;
31
- try {
32
- accountsList = loadAccounts();
33
- } catch (e) {
34
- // Fallback to JSON if CSV not found
35
- try {
36
- const jsonAccounts = JSON.parse(fs.readFileSync(path.join(configDir, 'accounts.json'), 'utf8'));
37
- accountsList = jsonAccounts.accounts.filter(a => a.active).map(a => ({
38
- account_id: a.id,
39
- brand: a.brand,
40
- platform: a.platforms ? a.platforms[0] : '',
41
- chrome_profile: a.chromeProfile,
42
- active: 'true',
43
- hasCredentials: false,
44
- }));
45
- // Expand multi-platform accounts
46
- const expanded = [];
47
- jsonAccounts.accounts.filter(a => a.active).forEach(a => {
48
- a.platforms.forEach(p => {
49
- expanded.push({
50
- account_id: `${a.id}-${p}`,
51
- brand: a.brand,
52
- brandSlug: a.brandSlug,
53
- platform: p,
54
- chrome_profile: a.chromeProfile,
55
- active: 'true',
56
- hasCredentials: false,
57
- });
58
- });
59
- });
60
- accountsList = expanded;
61
- } catch (e2) {
62
- console.error('❌ Cannot load accounts from CSV or JSON:', e.message);
63
- process.exit(1);
64
- }
65
- }
66
-
67
- const reportTypesConfig = JSON.parse(fs.readFileSync(path.join(configDir, 'report-types.json'), 'utf8'));
68
- const DEFAULT_OUTPUT_ROOT = path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'data');
69
-
70
- // --- CLI argument parsing ---
71
- const args = process.argv.slice(2);
72
- const getArg = (name) => {
73
- const idx = args.indexOf(`--${name}`);
74
- return idx !== -1 ? args[idx + 1] : null;
75
- };
76
- const hasFlag = (name) => args.includes(`--${name}`);
77
-
78
- // --- Commands ---
79
-
80
- if (hasFlag('list')) {
81
- // List all report types grouped by platform
82
- console.log('\n📋 Report Types Registry\n');
83
- const byPlatform = {};
84
- reportTypesConfig.reportTypes.forEach(rt => {
85
- if (!byPlatform[rt.platform]) byPlatform[rt.platform] = [];
86
- byPlatform[rt.platform].push(rt);
87
- });
88
-
89
- Object.entries(byPlatform).forEach(([platform, reports]) => {
90
- console.log(`\n🏪 ${platform.toUpperCase()} (${reports.length} types)`);
91
- console.log('─'.repeat(60));
92
- reports.forEach(r => {
93
- const status = r.status === 'active' ? '✅' : '🔄';
94
- const priority = r.priority ? '⭐' : ' ';
95
- console.log(` ${status} ${priority} ${r.id.padEnd(30)} ${r.frequency.padEnd(16)} ${r.exportFormat}`);
96
- });
97
- });
98
-
99
- console.log('\n\n👥 Accounts (from CSV):');
100
- const byBrand = {};
101
- accountsList.forEach(a => {
102
- if (!byBrand[a.brand]) byBrand[a.brand] = [];
103
- byBrand[a.brand].push(a.platform);
104
- });
105
- Object.entries(byBrand).forEach(([brand, platforms]) => {
106
- console.log(` • ${brand}: ${platforms.join(', ')} ${accountsList.find(a => a.brand === brand)?.hasCredentials ? '🔑' : '⚠️ no credentials'}`);
107
- });
108
-
109
- process.exit(0);
110
- }
111
-
112
- if (hasFlag('status')) {
113
- // Check download status for a period
114
- const outputRoot = args[args.indexOf('--status') + 1] || DEFAULT_OUTPUT_ROOT;
115
- const period = args[args.indexOf('--status') + 2] || getPreviousMonth();
116
-
117
- console.log(`\n📊 Download Status for ${period}\n`);
118
- console.log(`📂 Root: ${outputRoot}\n`);
119
-
120
- let found = 0;
121
- let missing = 0;
122
-
123
- const brandGroups = {};
124
- accountsList.forEach(a => {
125
- if (!brandGroups[a.brand]) brandGroups[a.brand] = { brand: a.brand, brandSlug: a.brand.replace(/ /g, '_'), platforms: [] };
126
- if (!brandGroups[a.brand].platforms.includes(a.platform)) brandGroups[a.brand].platforms.push(a.platform);
127
- });
128
- Object.values(brandGroups).forEach(account => {
129
- console.log(`\n👤 ${account.brand}`);
130
- account.platforms.forEach(platform => {
131
- const reports = getReportsForPlatform(platform);
132
- reports.forEach(rt => {
133
- const platformDir = platform.charAt(0).toUpperCase() + platform.slice(1);
134
- const filePath = path.join(
135
- outputRoot, platformDir, account.brandSlug, rt.id,
136
- `${period}_${account.brandSlug}_${rt.id}.${rt.fileExtension}`
137
- );
138
-
139
- if (fs.existsSync(filePath)) {
140
- const stats = fs.statSync(filePath);
141
- const sizeKb = (stats.size / 1024).toFixed(1);
142
- console.log(` ✅ ${platform}/${rt.id} (${sizeKb} KB)`);
143
- found++;
144
- } else {
145
- console.log(` ❌ ${platform}/${rt.id}`);
146
- missing++;
147
- }
148
- });
149
- });
150
- });
151
-
152
- console.log(`\n📊 Summary: ${found} found, ${missing} missing\n`);
153
- process.exit(0);
154
- }
155
-
156
- if (hasFlag('plan')) {
157
- // Generate a crawl plan for the browser agent
158
- const platform = getArg('platform');
159
- const brand = getArg('brand');
160
- const period = getArg('period') || getPreviousMonth();
161
- const outputRoot = getArg('output') || DEFAULT_OUTPUT_ROOT;
162
-
163
- console.log('\n🗺️ Crawl Plan\n');
164
- console.log(`Platform: ${platform || 'ALL'}`);
165
- console.log(`Brand: ${brand || 'ALL'}`);
166
- console.log(`Period: ${period}`);
167
- console.log(`Output: ${outputRoot}\n`);
168
-
169
- const tasks = generateCrawlTasks(platform, brand, period, outputRoot);
170
-
171
- tasks.forEach((task, i) => {
172
- console.log(`\n--- Task ${i + 1}/${tasks.length} ---`);
173
- console.log(`Platform: ${task.platform}`);
174
- console.log(`Brand: ${task.brand}`);
175
- console.log(`Report: ${task.reportType.name} (${task.reportType.id})`);
176
- console.log(`URL: ${task.reportType.url}`);
177
- console.log(`Save to: ${task.outputPath}`);
178
- console.log(`Steps:`);
179
- task.reportType.exportSteps.forEach((step, j) => {
180
- console.log(` ${j + 1}. ${step}`);
181
- });
182
- });
183
-
184
- console.log(`\n📋 Total tasks: ${tasks.length}`);
185
- process.exit(0);
186
- }
187
-
188
- if (hasFlag('dashboard')) {
189
- // Open dashboard in browser
190
- const dashboardPath = path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'dashboard', 'index.html');
191
- const { exec } = require('child_process');
192
- const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'start' : 'xdg-open';
193
- exec(`${cmd} "${dashboardPath}"`);
194
- console.log(`\n📊 Opening dashboard: ${dashboardPath}\n`);
195
- logger.activity({ action: 'open_dashboard', message: 'Dashboard opened' });
196
- process.exit(0);
197
- }
198
-
199
- // --- Default: show help ---
200
- console.log(`
201
- 🕷️ Marketplace Report Crawl Runner
202
- ====================================
203
-
204
- Commands:
205
- --list List all report types & accounts
206
- --status <output_root> <YYYYMM> Check download status
207
- --plan [options] Generate crawl plan for browser agent
208
- --dashboard Open dashboard in browser
209
-
210
- Plan options:
211
- --platform <shopee|lazada|tiktok> Filter by platform
212
- --brand <brand_name> Filter by brand
213
- --period <YYYYMM> Target period (default: previous month)
214
- --output <path> Output root directory
215
-
216
- Examples:
217
- node crawl-runner.js --list
218
- node crawl-runner.js --status ./data 202602
219
- node crawl-runner.js --plan --platform shopee --brand "Vita Dairy" --period 202602
220
- node crawl-runner.js --plan --platform lazada --period 202602
221
- node crawl-runner.js --dashboard
222
- `);
223
-
224
- // --- Helper functions ---
225
-
226
- function getPreviousMonth() {
227
- const d = new Date();
228
- d.setMonth(d.getMonth() - 1);
229
- return d.getFullYear().toString() + (d.getMonth() + 1).toString().padStart(2, '0');
230
- }
231
-
232
- function getReportsForPlatform(platform) {
233
- const seen = new Set();
234
- return reportTypesConfig.reportTypes.filter(rt => {
235
- if (rt.platform !== platform || rt.status === 'disabled') return false;
236
- if (seen.has(rt.id)) return false;
237
- seen.add(rt.id);
238
- return true;
239
- });
240
- }
241
-
242
- function generateCrawlTasks(platform, brand, period, outputRoot) {
243
- const tasks = [];
244
- const brandGroups = {};
245
- accountsList.forEach(a => {
246
- if (!brandGroups[a.brand]) brandGroups[a.brand] = { brand: a.brand, brandSlug: a.brand.replace(/ /g, '_'), platforms: [], chromeProfile: a.chrome_profile };
247
- if (!brandGroups[a.brand].platforms.includes(a.platform)) brandGroups[a.brand].platforms.push(a.platform);
248
- });
249
- const targetAccounts = Object.values(brandGroups).filter(a => {
250
- if (brand && a.brand !== brand) return false;
251
- return true;
252
- });
253
-
254
- targetAccounts.forEach(account => {
255
- const targetPlatforms = platform ? [platform] : account.platforms;
256
- targetPlatforms.forEach(p => {
257
- if (!account.platforms.includes(p)) return;
258
- const reports = getReportsForPlatform(p);
259
- reports.forEach(rt => {
260
- const platformDir = p.charAt(0).toUpperCase() + p.slice(1);
261
- const outputPath = path.join(
262
- outputRoot, platformDir, account.brandSlug, rt.id,
263
- `${period}_${account.brandSlug}_${rt.id}.${rt.fileExtension}`
264
- );
265
- tasks.push({
266
- platform: p,
267
- brand: account.brand,
268
- brandSlug: account.brandSlug,
269
- chromeProfile: account.chromeProfile,
270
- reportType: rt,
271
- period,
272
- outputPath
273
- });
274
- });
275
- });
276
- });
277
-
278
- return tasks;
279
- }
280
-
281
- module.exports = { generateCrawlTasks, getReportsForPlatform, getPreviousMonth };
@@ -1,72 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # Session Health Check — Validates if a marketplace session is still active
4
- # =============================================================================
5
- # Returns exit code 0 if session is valid, 1 if expired/invalid.
6
- # Designed to be called by other scripts or directly by AI agents.
7
- #
8
- # Usage:
9
- # ./session-check.sh <platform> <brand-id>
10
- # # Exit 0 = valid, Exit 1 = expired/no-profile
11
- # =============================================================================
12
-
13
- set -uo pipefail
14
-
15
- PROFILE_ROOT="${MARKETPLACE_PROFILE_ROOT:-$HOME/.marketplace-crawler/profiles}"
16
- AB_CMD="${AGENT_BROWSER_CMD:-npx -y agent-browser}"
17
-
18
- platform="${1:-}"
19
- brand_id="${2:-}"
20
-
21
- if [ -z "$platform" ] || [ -z "$brand_id" ]; then
22
- echo "Usage: $0 <platform> <brand-id>"
23
- echo "Example: $0 shopee vita-dairy"
24
- exit 2
25
- fi
26
-
27
- PROFILE_DIR="$PROFILE_ROOT/${platform}-${brand_id}"
28
-
29
- # Check if profile exists
30
- if [ ! -d "$PROFILE_DIR" ]; then
31
- echo "NO_PROFILE"
32
- exit 1
33
- fi
34
-
35
- # Define test URLs and login page indicators
36
- case "$platform" in
37
- shopee)
38
- CHECK_URL="https://banhang.shopee.vn/portal/dashboard/"
39
- LOGIN_PATTERN="buyer/login\|account/login"
40
- ;;
41
- lazada)
42
- CHECK_URL="https://sellercenter.lazada.vn/apps/dashboard"
43
- LOGIN_PATTERN="seller/login\|sso/login"
44
- ;;
45
- tiktok)
46
- CHECK_URL="https://seller-vn.tiktok.com/homepage?shop_region=VN"
47
- LOGIN_PATTERN="account/login\|passport"
48
- ;;
49
- *)
50
- echo "UNKNOWN_PLATFORM"
51
- exit 2
52
- ;;
53
- esac
54
-
55
- # Try to navigate to dashboard and check if redirected to login
56
- $AB_CMD --profile "$PROFILE_DIR" open "$CHECK_URL" 2>/dev/null
57
- sleep 3
58
- $AB_CMD --profile "$PROFILE_DIR" wait --load networkidle 2>/dev/null || true
59
-
60
- CURRENT_URL=$($AB_CMD --profile "$PROFILE_DIR" get url 2>/dev/null || echo "ERROR")
61
- $AB_CMD --profile "$PROFILE_DIR" close 2>/dev/null || true
62
-
63
- if echo "$CURRENT_URL" | grep -qi "$LOGIN_PATTERN"; then
64
- echo "EXPIRED"
65
- exit 1
66
- elif echo "$CURRENT_URL" | grep -qi "error\|ERROR"; then
67
- echo "ERROR"
68
- exit 1
69
- else
70
- echo "VALID"
71
- exit 0
72
- fi
@@ -1,349 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # Session Manager for Marketplace Report Crawler
4
- # =============================================================================
5
- # Cross-agent compatible: works from ANY AI coding agent that can run shell.
6
- # Uses agent-browser CLI for persistent browser sessions.
7
- #
8
- # Usage:
9
- # ./session-manager.sh status # Show all sessions
10
- # ./session-manager.sh login <platform> <brand-id> # Guided login (headed)
11
- # ./session-manager.sh login-all # Login all active accounts
12
- # ./session-manager.sh check <platform> <brand-id> # Check session health
13
- # ./session-manager.sh check-all # Check all sessions
14
- # =============================================================================
15
-
16
- set -euo pipefail
17
-
18
- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
19
- CONFIG_DIR="$SCRIPT_DIR/../config"
20
- ACCOUNTS_FILE="$CONFIG_DIR/accounts.json"
21
- SESSIONS_FILE="$CONFIG_DIR/sessions.json"
22
-
23
- # Default profile root — can be overridden via env
24
- PROFILE_ROOT="${MARKETPLACE_PROFILE_ROOT:-$HOME/.marketplace-crawler/profiles}"
25
-
26
- # agent-browser command — use npx if not installed globally
27
- AB_CMD="${AGENT_BROWSER_CMD:-npx -y agent-browser}"
28
-
29
- # Platform URLs via functions (compatible with bash 3.x on macOS)
30
- get_login_url() {
31
- case "$1" in
32
- shopee) echo "https://banhang.shopee.vn/" ;;
33
- lazada) echo "https://sellercenter.lazada.vn/" ;;
34
- tiktok) echo "https://seller-vn.tiktok.com/" ;;
35
- *) echo "" ;;
36
- esac
37
- }
38
-
39
- get_check_url() {
40
- case "$1" in
41
- shopee) echo "https://banhang.shopee.vn/portal/dashboard/" ;;
42
- lazada) echo "https://sellercenter.lazada.vn/apps/dashboard" ;;
43
- tiktok) echo "https://seller-vn.tiktok.com/homepage?shop_region=VN" ;;
44
- *) echo "" ;;
45
- esac
46
- }
47
-
48
- get_login_indicator() {
49
- case "$1" in
50
- shopee) echo "buyer/login" ;;
51
- lazada) echo "seller/login" ;;
52
- tiktok) echo "account/login" ;;
53
- *) echo "login" ;;
54
- esac
55
- }
56
-
57
- # --- Helpers ---
58
-
59
- get_profile_dir() {
60
- local platform="$1"
61
- local brand_id="$2"
62
- echo "$PROFILE_ROOT/${platform}-${brand_id}"
63
- }
64
-
65
- ensure_profile_dir() {
66
- local dir="$1"
67
- mkdir -p "$dir"
68
- }
69
-
70
- get_brands() {
71
- # Returns brand IDs from accounts.json
72
- node -e "
73
- const a = require('$ACCOUNTS_FILE');
74
- a.accounts.filter(x => x.active).forEach(x => console.log(x.id));
75
- "
76
- }
77
-
78
- get_brand_platforms() {
79
- local brand_id="$1"
80
- node -e "
81
- const a = require('$ACCOUNTS_FILE');
82
- const b = a.accounts.find(x => x.id === '$brand_id');
83
- if (b) b.platforms.forEach(p => console.log(p));
84
- "
85
- }
86
-
87
- get_brand_name() {
88
- local brand_id="$1"
89
- node -e "
90
- const a = require('$ACCOUNTS_FILE');
91
- const b = a.accounts.find(x => x.id === '$brand_id');
92
- if (b) console.log(b.brand);
93
- "
94
- }
95
-
96
- update_session_status() {
97
- local platform="$1"
98
- local brand_id="$2"
99
- local status="$3"
100
- local now
101
- now=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
102
-
103
- # Create sessions.json if not exists
104
- if [ ! -f "$SESSIONS_FILE" ]; then
105
- echo '{"sessions":[]}' > "$SESSIONS_FILE"
106
- fi
107
-
108
- node -e "
109
- const fs = require('fs');
110
- const data = JSON.parse(fs.readFileSync('$SESSIONS_FILE', 'utf8'));
111
- const idx = data.sessions.findIndex(s => s.platform === '$platform' && s.brandId === '$brand_id');
112
- const entry = {
113
- platform: '$platform',
114
- brandId: '$brand_id',
115
- status: '$status',
116
- profilePath: '$(get_profile_dir "$platform" "$brand_id")',
117
- $([ "$status" = "valid" ] && echo "lastLogin: '$now'," || echo "")
118
- lastCheck: '$now'
119
- };
120
- if (idx >= 0) { Object.assign(data.sessions[idx], entry); }
121
- else { data.sessions.push(entry); }
122
- fs.writeFileSync('$SESSIONS_FILE', JSON.stringify(data, null, 2));
123
- "
124
- }
125
-
126
- # --- Commands ---
127
-
128
- cmd_status() {
129
- echo ""
130
- echo "📊 Marketplace Session Status"
131
- echo "═══════════════════════════════════════════════════════"
132
- echo ""
133
- echo "Profile root: $PROFILE_ROOT"
134
- echo ""
135
-
136
- if [ ! -f "$SESSIONS_FILE" ]; then
137
- echo "⚠️ No sessions tracked yet. Run 'login-all' first."
138
- return
139
- fi
140
-
141
- node -e "
142
- const fs = require('fs');
143
- const data = JSON.parse(fs.readFileSync('$SESSIONS_FILE', 'utf8'));
144
- if (data.sessions.length === 0) {
145
- console.log('⚠️ No sessions tracked yet.');
146
- process.exit(0);
147
- }
148
- const icons = { valid: '✅', expired: '❌', unknown: '❓' };
149
- data.sessions.forEach(s => {
150
- const icon = icons[s.status] || '❓';
151
- const last = s.lastCheck ? new Date(s.lastCheck).toLocaleString() : 'never';
152
- console.log(\` \${icon} \${s.platform.padEnd(8)} \${s.brandId.padEnd(15)} \${s.status.padEnd(10)} last: \${last}\`);
153
- });
154
- "
155
- echo ""
156
- }
157
-
158
- cmd_login() {
159
- local platform="$1"
160
- local brand_id="$2"
161
- local brand_name
162
- brand_name=$(get_brand_name "$brand_id")
163
- local profile_dir
164
- profile_dir=$(get_profile_dir "$platform" "$brand_id")
165
- local login_url
166
- login_url=$(get_login_url "$platform")
167
-
168
- ensure_profile_dir "$profile_dir"
169
-
170
- echo ""
171
- echo "🔐 Guided Login: $brand_name on ${platform^^}"
172
- echo "═══════════════════════════════════════════════════════"
173
- echo ""
174
- echo "📂 Profile: $profile_dir"
175
- echo "🌐 URL: $login_url"
176
- echo ""
177
- echo "⏳ Opening browser in headed mode..."
178
- echo " → Please login manually (solve CAPTCHA/OTP as needed)"
179
- echo " → When you see the dashboard, press Enter here to save session"
180
- echo ""
181
-
182
- # Open browser in headed mode with persistent profile
183
- $AB_CMD --profile "$profile_dir" --headed open "$login_url" &
184
- local ab_pid=$!
185
-
186
- # Wait for user to confirm login
187
- read -r -p "✅ Press Enter when login is complete... "
188
-
189
- # Take a screenshot as proof
190
- local screenshot_dir="$profile_dir/screenshots"
191
- mkdir -p "$screenshot_dir"
192
- $AB_CMD --profile "$profile_dir" screenshot "$screenshot_dir/login-success-$(date +%Y%m%d).png" 2>/dev/null || true
193
-
194
- # Close the browser (profile is auto-saved)
195
- $AB_CMD --profile "$profile_dir" close 2>/dev/null || true
196
-
197
- # Update session tracking
198
- update_session_status "$platform" "$brand_id" "valid"
199
-
200
- echo ""
201
- echo "✅ Session saved for $brand_name on ${platform^^}"
202
- echo " Profile: $profile_dir"
203
- echo ""
204
- }
205
-
206
- cmd_login_all() {
207
- echo ""
208
- echo "🔐 Guided Login — All Active Accounts"
209
- echo "═══════════════════════════════════════════════════════"
210
- echo ""
211
-
212
- local brands
213
- brands=$(get_brands)
214
-
215
- for brand_id in $brands; do
216
- local platforms
217
- platforms=$(get_brand_platforms "$brand_id")
218
- for platform in $platforms; do
219
- cmd_login "$platform" "$brand_id"
220
- done
221
- done
222
-
223
- echo ""
224
- echo "✅ All accounts logged in. Run 'check-all' to verify."
225
- echo ""
226
- }
227
-
228
- cmd_check() {
229
- local platform="$1"
230
- local brand_id="$2"
231
- local brand_name
232
- brand_name=$(get_brand_name "$brand_id")
233
- local profile_dir
234
- profile_dir=$(get_profile_dir "$platform" "$brand_id")
235
- local check_url
236
- check_url=$(get_check_url "$platform")
237
- local login_indicator
238
- login_indicator=$(get_login_indicator "$platform")
239
-
240
- if [ ! -d "$profile_dir" ]; then
241
- echo "❌ ${platform}/${brand_id}: No profile found (never logged in)"
242
- update_session_status "$platform" "$brand_id" "unknown"
243
- return 1
244
- fi
245
-
246
- # Open in headless mode, navigate to dashboard, check URL
247
- $AB_CMD --profile "$profile_dir" open "$check_url" 2>/dev/null || true
248
- sleep 3
249
- $AB_CMD --profile "$profile_dir" wait --load networkidle 2>/dev/null || true
250
-
251
- local current_url
252
- current_url=$($AB_CMD --profile "$profile_dir" get url 2>/dev/null || echo "error")
253
-
254
- $AB_CMD --profile "$profile_dir" close 2>/dev/null || true
255
-
256
- if echo "$current_url" | grep -qi "$login_indicator"; then
257
- echo "❌ ${platform}/${brand_id} ($brand_name): Session EXPIRED"
258
- update_session_status "$platform" "$brand_id" "expired"
259
- return 1
260
- else
261
- echo "✅ ${platform}/${brand_id} ($brand_name): Session VALID"
262
- update_session_status "$platform" "$brand_id" "valid"
263
- return 0
264
- fi
265
- }
266
-
267
- cmd_check_all() {
268
- echo ""
269
- echo "🔍 Checking All Sessions"
270
- echo "═══════════════════════════════════════════════════════"
271
- echo ""
272
-
273
- local valid=0
274
- local expired=0
275
- local unknown=0
276
-
277
- local brands
278
- brands=$(get_brands)
279
-
280
- for brand_id in $brands; do
281
- local platforms
282
- platforms=$(get_brand_platforms "$brand_id")
283
- for platform in $platforms; do
284
- if cmd_check "$platform" "$brand_id"; then
285
- ((valid++))
286
- else
287
- ((expired++))
288
- fi
289
- done
290
- done
291
-
292
- echo ""
293
- echo "📊 Summary: $valid valid, $expired expired/unknown"
294
- echo ""
295
- }
296
-
297
- # --- Main ---
298
-
299
- case "${1:-help}" in
300
- status)
301
- cmd_status
302
- ;;
303
- login)
304
- if [ $# -lt 3 ]; then
305
- echo "Usage: $0 login <platform> <brand-id>"
306
- echo "Platforms: shopee, lazada, tiktok"
307
- exit 1
308
- fi
309
- cmd_login "$2" "$3"
310
- ;;
311
- login-all)
312
- cmd_login_all
313
- ;;
314
- check)
315
- if [ $# -lt 3 ]; then
316
- echo "Usage: $0 check <platform> <brand-id>"
317
- exit 1
318
- fi
319
- cmd_check "$2" "$3"
320
- ;;
321
- check-all)
322
- cmd_check_all
323
- ;;
324
- *)
325
- echo ""
326
- echo "🕷️ Marketplace Session Manager"
327
- echo "════════════════════════════════"
328
- echo ""
329
- echo "Commands:"
330
- echo " status Show all session statuses"
331
- echo " login <platform> <brand-id> Guided login (opens browser)"
332
- echo " login-all Login all active accounts"
333
- echo " check <platform> <brand-id> Check if session is valid"
334
- echo " check-all Check all sessions"
335
- echo ""
336
- echo "Platforms: shopee, lazada, tiktok"
337
- echo "Brands: $(get_brands | tr '\n' ', ')"
338
- echo ""
339
- echo "Environment:"
340
- echo " MARKETPLACE_PROFILE_ROOT Profile directory (default: ~/.marketplace-crawler/profiles)"
341
- echo " AGENT_BROWSER_CMD agent-browser command (default: npx -y agent-browser)"
342
- echo ""
343
- echo "Examples:"
344
- echo " $0 login shopee vita-dairy"
345
- echo " $0 check-all"
346
- echo " $0 status"
347
- echo ""
348
- ;;
349
- esac