codymaster 4.4.4 → 4.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +29 -14
  3. package/commands/demo.md +1 -1
  4. package/dist/context-bus.js +70 -0
  5. package/dist/context-db.js +265 -0
  6. package/dist/continuity.js +12 -0
  7. package/dist/file-watcher.js +79 -0
  8. package/dist/index.js +152 -1
  9. package/dist/l0-indexer.js +158 -0
  10. package/dist/mcp-context-server.js +400 -0
  11. package/dist/migrate-json-to-sqlite.js +126 -0
  12. package/dist/skill-chain.js +19 -3
  13. package/dist/token-budget.js +108 -0
  14. package/dist/uri-resolver.js +203 -0
  15. package/package.json +7 -1
  16. package/skills/_shared/helpers.md +50 -14
  17. package/skills/cm-autopilot/SKILL.md +29 -0
  18. package/skills/cm-autopilot/scripts/autopilot.py +190 -0
  19. package/skills/cm-continuity/SKILL.md +90 -28
  20. package/skills/cm-quality-gate/SKILL.md +11 -1
  21. package/skills/cm-safe-deploy/SKILL.md +38 -2
  22. package/skills/cm-security-gate/SKILL.md +158 -34
  23. package/skills/cm-skill-chain/SKILL.md +47 -1
  24. package/skills/cm-start/SKILL.md +11 -2
  25. package/skills/cm-test-gate/SKILL.md +3 -0
  26. package/skills/boxme-git-config/SKILL.md +0 -56
  27. package/skills/boxme-local-dev/SKILL.md +0 -66
  28. package/skills/jobs-to-be-done/SKILL.md +0 -266
  29. package/skills/jobs-to-be-done/references/case-studies.md +0 -154
  30. package/skills/jobs-to-be-done/references/competitive-strategy.md +0 -280
  31. package/skills/jobs-to-be-done/references/diagnostics.md +0 -158
  32. package/skills/jobs-to-be-done/references/innovation-process.md +0 -392
  33. package/skills/jobs-to-be-done/references/organizational-change.md +0 -328
  34. package/skills/marketplace-report-crawler/SKILL.md +0 -176
  35. package/skills/marketplace-report-crawler/config/accounts.json +0 -41
  36. package/skills/marketplace-report-crawler/config/report-types.json +0 -422
  37. package/skills/marketplace-report-crawler/config/sessions.json +0 -3
  38. package/skills/marketplace-report-crawler/scripts/ab-wrapper.sh +0 -102
  39. package/skills/marketplace-report-crawler/scripts/browser-actions/lazada/lazada-actions.js +0 -114
  40. package/skills/marketplace-report-crawler/scripts/browser-actions/shopee/shopee-actions.js +0 -94
  41. package/skills/marketplace-report-crawler/scripts/browser-actions/tiktok/tiktok-actions.js +0 -272
  42. package/skills/marketplace-report-crawler/scripts/crawl-runner.js +0 -281
  43. package/skills/marketplace-report-crawler/scripts/session-check.sh +0 -72
  44. package/skills/marketplace-report-crawler/scripts/session-manager.sh +0 -349
  45. package/skills/marketplace-report-crawler/scripts/setup-folders.sh +0 -83
  46. package/skills/medical-research/SKILL.md +0 -194
  47. package/skills/medical-research/scripts/evidence_checker.py +0 -288
  48. package/skills/mom-test/SKILL.md +0 -267
  49. package/skills/mom-test/references/avoiding-bad-data.md +0 -221
  50. package/skills/mom-test/references/case-studies.md +0 -306
  51. package/skills/mom-test/references/commitment-advancement.md +0 -219
  52. package/skills/mom-test/references/finding-conversations.md +0 -251
  53. package/skills/mom-test/references/processing-learning.md +0 -256
  54. package/skills/mom-test/references/question-patterns.md +0 -198
  55. package/skills/pandasai-analytics/SKILL.md +0 -251
  56. package/skills/release-it/SKILL.md +0 -235
  57. package/skills/release-it/references/anti-patterns.md +0 -279
  58. package/skills/release-it/references/capacity-planning.md +0 -285
  59. package/skills/release-it/references/chaos-engineering.md +0 -325
  60. package/skills/release-it/references/deployment-strategies.md +0 -331
  61. package/skills/release-it/references/observability.md +0 -301
  62. package/skills/release-it/references/stability-patterns.md +0 -355
  63. package/skills/skill-creator-ultra/.agents/workflows/skill-audit.md +0 -37
  64. package/skills/skill-creator-ultra/.agents/workflows/skill-compare.md +0 -34
  65. package/skills/skill-creator-ultra/.agents/workflows/skill-export.md +0 -51
  66. package/skills/skill-creator-ultra/.agents/workflows/skill-generate.md +0 -39
  67. package/skills/skill-creator-ultra/.agents/workflows/skill-scaffold.md +0 -52
  68. package/skills/skill-creator-ultra/.agents/workflows/skill-simulate.md +0 -25
  69. package/skills/skill-creator-ultra/.agents/workflows/skill-stats.md +0 -31
  70. package/skills/skill-creator-ultra/.agents/workflows/skill-validate.md +0 -25
  71. package/skills/skill-creator-ultra/README.md +0 -1242
  72. package/skills/skill-creator-ultra/SKILL.md +0 -388
  73. package/skills/skill-creator-ultra/agents/analyzer.md +0 -274
  74. package/skills/skill-creator-ultra/agents/comparator.md +0 -202
  75. package/skills/skill-creator-ultra/agents/grader.md +0 -223
  76. package/skills/skill-creator-ultra/assets/eval_review.html +0 -146
  77. package/skills/skill-creator-ultra/eval-viewer/generate_review.py +0 -471
  78. package/skills/skill-creator-ultra/eval-viewer/viewer.html +0 -1325
  79. package/skills/skill-creator-ultra/examples/example_anthropic_frontend.md +0 -109
  80. package/skills/skill-creator-ultra/examples/example_anthropic_pdf.md +0 -116
  81. package/skills/skill-creator-ultra/examples/example_api_docs.md +0 -189
  82. package/skills/skill-creator-ultra/examples/example_db_migration.md +0 -253
  83. package/skills/skill-creator-ultra/examples/example_git_commit.md +0 -111
  84. package/skills/skill-creator-ultra/install.ps1 +0 -289
  85. package/skills/skill-creator-ultra/install.sh +0 -313
  86. package/skills/skill-creator-ultra/phases/phase1_interview.md +0 -202
  87. package/skills/skill-creator-ultra/phases/phase2_extract.md +0 -55
  88. package/skills/skill-creator-ultra/phases/phase3_detect.md +0 -57
  89. package/skills/skill-creator-ultra/phases/phase4_generate.md +0 -543
  90. package/skills/skill-creator-ultra/phases/phase5_test.md +0 -319
  91. package/skills/skill-creator-ultra/phases/phase6_eval.md +0 -301
  92. package/skills/skill-creator-ultra/phases/phase7_iterate.md +0 -103
  93. package/skills/skill-creator-ultra/phases/phase8_optimize.md +0 -113
  94. package/skills/skill-creator-ultra/resources/advanced_patterns.md +0 -499
  95. package/skills/skill-creator-ultra/resources/anti_patterns.md +0 -376
  96. package/skills/skill-creator-ultra/resources/blueprints.md +0 -498
  97. package/skills/skill-creator-ultra/resources/checklist.md +0 -243
  98. package/skills/skill-creator-ultra/resources/composition_cookbook.md +0 -291
  99. package/skills/skill-creator-ultra/resources/description_optimization.md +0 -90
  100. package/skills/skill-creator-ultra/resources/eval_guide.md +0 -133
  101. package/skills/skill-creator-ultra/resources/industry_questions.md +0 -189
  102. package/skills/skill-creator-ultra/resources/interview_questions.md +0 -200
  103. package/skills/skill-creator-ultra/resources/pattern_detection.md +0 -200
  104. package/skills/skill-creator-ultra/resources/prompt_engineering.md +0 -531
  105. package/skills/skill-creator-ultra/resources/schemas.md +0 -430
  106. package/skills/skill-creator-ultra/resources/script_integration.md +0 -593
  107. package/skills/skill-creator-ultra/resources/scripts_guide.md +0 -339
  108. package/skills/skill-creator-ultra/resources/skill_template.md +0 -124
  109. package/skills/skill-creator-ultra/resources/skill_writing_guide.md +0 -634
  110. package/skills/skill-creator-ultra/resources/versioning_guide.md +0 -193
  111. package/skills/skill-creator-ultra/scripts/ci_eval.py +0 -200
  112. package/skills/skill-creator-ultra/scripts/package_skill.py +0 -165
  113. package/skills/skill-creator-ultra/scripts/simulate_skill.py +0 -398
  114. package/skills/skill-creator-ultra/scripts/skill_audit.py +0 -611
  115. package/skills/skill-creator-ultra/scripts/skill_compare.py +0 -265
  116. package/skills/skill-creator-ultra/scripts/skill_export.py +0 -334
  117. package/skills/skill-creator-ultra/scripts/skill_scaffold.py +0 -403
  118. package/skills/skill-creator-ultra/scripts/skill_stats.py +0 -339
  119. package/skills/skill-creator-ultra/scripts/validate_skill.py +0 -411
  120. package/skills/tailwind-mastery/SKILL.md +0 -229
  121. package/skills/vercel-react-best-practices/AGENTS.md +0 -3373
  122. package/skills/vercel-react-best-practices/README.md +0 -123
  123. package/skills/vercel-react-best-practices/SKILL.md +0 -143
  124. package/skills/vercel-react-best-practices/rules/_sections.md +0 -46
  125. package/skills/vercel-react-best-practices/rules/_template.md +0 -28
  126. package/skills/vercel-react-best-practices/rules/advanced-event-handler-refs.md +0 -55
  127. package/skills/vercel-react-best-practices/rules/advanced-init-once.md +0 -42
  128. package/skills/vercel-react-best-practices/rules/advanced-use-latest.md +0 -39
  129. package/skills/vercel-react-best-practices/rules/async-api-routes.md +0 -38
  130. package/skills/vercel-react-best-practices/rules/async-defer-await.md +0 -80
  131. package/skills/vercel-react-best-practices/rules/async-dependencies.md +0 -51
  132. package/skills/vercel-react-best-practices/rules/async-parallel.md +0 -28
  133. package/skills/vercel-react-best-practices/rules/async-suspense-boundaries.md +0 -99
  134. package/skills/vercel-react-best-practices/rules/bundle-barrel-imports.md +0 -59
  135. package/skills/vercel-react-best-practices/rules/bundle-conditional.md +0 -31
  136. package/skills/vercel-react-best-practices/rules/bundle-defer-third-party.md +0 -49
  137. package/skills/vercel-react-best-practices/rules/bundle-dynamic-imports.md +0 -35
  138. package/skills/vercel-react-best-practices/rules/bundle-preload.md +0 -50
  139. package/skills/vercel-react-best-practices/rules/client-event-listeners.md +0 -74
  140. package/skills/vercel-react-best-practices/rules/client-localstorage-schema.md +0 -71
  141. package/skills/vercel-react-best-practices/rules/client-passive-event-listeners.md +0 -48
  142. package/skills/vercel-react-best-practices/rules/client-swr-dedup.md +0 -56
  143. package/skills/vercel-react-best-practices/rules/js-batch-dom-css.md +0 -107
  144. package/skills/vercel-react-best-practices/rules/js-cache-function-results.md +0 -80
  145. package/skills/vercel-react-best-practices/rules/js-cache-property-access.md +0 -28
  146. package/skills/vercel-react-best-practices/rules/js-cache-storage.md +0 -70
  147. package/skills/vercel-react-best-practices/rules/js-combine-iterations.md +0 -32
  148. package/skills/vercel-react-best-practices/rules/js-early-exit.md +0 -50
  149. package/skills/vercel-react-best-practices/rules/js-flatmap-filter.md +0 -60
  150. package/skills/vercel-react-best-practices/rules/js-hoist-regexp.md +0 -45
  151. package/skills/vercel-react-best-practices/rules/js-index-maps.md +0 -37
  152. package/skills/vercel-react-best-practices/rules/js-length-check-first.md +0 -49
  153. package/skills/vercel-react-best-practices/rules/js-min-max-loop.md +0 -82
  154. package/skills/vercel-react-best-practices/rules/js-set-map-lookups.md +0 -24
  155. package/skills/vercel-react-best-practices/rules/js-tosorted-immutable.md +0 -57
  156. package/skills/vercel-react-best-practices/rules/rendering-activity.md +0 -26
  157. package/skills/vercel-react-best-practices/rules/rendering-animate-svg-wrapper.md +0 -47
  158. package/skills/vercel-react-best-practices/rules/rendering-conditional-render.md +0 -40
  159. package/skills/vercel-react-best-practices/rules/rendering-content-visibility.md +0 -38
  160. package/skills/vercel-react-best-practices/rules/rendering-hoist-jsx.md +0 -46
  161. package/skills/vercel-react-best-practices/rules/rendering-hydration-no-flicker.md +0 -82
  162. package/skills/vercel-react-best-practices/rules/rendering-hydration-suppress-warning.md +0 -30
  163. package/skills/vercel-react-best-practices/rules/rendering-resource-hints.md +0 -85
  164. package/skills/vercel-react-best-practices/rules/rendering-script-defer-async.md +0 -68
  165. package/skills/vercel-react-best-practices/rules/rendering-svg-precision.md +0 -28
  166. package/skills/vercel-react-best-practices/rules/rendering-usetransition-loading.md +0 -75
  167. package/skills/vercel-react-best-practices/rules/rerender-defer-reads.md +0 -39
  168. package/skills/vercel-react-best-practices/rules/rerender-dependencies.md +0 -45
  169. package/skills/vercel-react-best-practices/rules/rerender-derived-state-no-effect.md +0 -40
  170. package/skills/vercel-react-best-practices/rules/rerender-derived-state.md +0 -29
  171. package/skills/vercel-react-best-practices/rules/rerender-functional-setstate.md +0 -74
  172. package/skills/vercel-react-best-practices/rules/rerender-lazy-state-init.md +0 -58
  173. package/skills/vercel-react-best-practices/rules/rerender-memo-with-default-value.md +0 -38
  174. package/skills/vercel-react-best-practices/rules/rerender-memo.md +0 -44
  175. package/skills/vercel-react-best-practices/rules/rerender-move-effect-to-event.md +0 -45
  176. package/skills/vercel-react-best-practices/rules/rerender-no-inline-components.md +0 -82
  177. package/skills/vercel-react-best-practices/rules/rerender-simple-expression-in-memo.md +0 -35
  178. package/skills/vercel-react-best-practices/rules/rerender-split-combined-hooks.md +0 -64
  179. package/skills/vercel-react-best-practices/rules/rerender-transitions.md +0 -40
  180. package/skills/vercel-react-best-practices/rules/rerender-use-deferred-value.md +0 -59
  181. package/skills/vercel-react-best-practices/rules/rerender-use-ref-transient-values.md +0 -73
  182. package/skills/vercel-react-best-practices/rules/server-after-nonblocking.md +0 -73
  183. package/skills/vercel-react-best-practices/rules/server-auth-actions.md +0 -96
  184. package/skills/vercel-react-best-practices/rules/server-cache-lru.md +0 -41
  185. package/skills/vercel-react-best-practices/rules/server-cache-react.md +0 -76
  186. package/skills/vercel-react-best-practices/rules/server-dedup-props.md +0 -65
  187. package/skills/vercel-react-best-practices/rules/server-hoist-static-io.md +0 -142
  188. package/skills/vercel-react-best-practices/rules/server-parallel-fetching.md +0 -83
  189. package/skills/vercel-react-best-practices/rules/server-serialization.md +0 -38
  190. package/skills/web-design-guidelines/SKILL.md +0 -39
@@ -1,281 +0,0 @@
1
- /**
2
- * Marketplace Report Crawl Runner
3
- * ================================
4
- * Orchestrator that reads config files and generates crawl instructions
5
- * for the browser agent to execute.
6
- *
7
- * UPDATED: Now reads from accounts.csv (via csv-reader.js) instead of accounts.json
8
- * Includes logging integration for activity & bug tracking.
9
- *
10
- * Usage:
11
- * node crawl-runner.js --platform shopee --brand "Vita Dairy" --period 202602
12
- * node crawl-runner.js --list # List all report types
13
- * node crawl-runner.js --setup-folders ./data # Generate folder structure
14
- * node crawl-runner.js --status ./data 202602 # Check download status
15
- * node crawl-runner.js --dashboard # Open dashboard
16
- */
17
-
18
- const fs = require('fs');
19
- const path = require('path');
20
-
21
- // --- Load configs ---
22
- const configDir = path.join(__dirname, '..', 'config');
23
- const { loadAccounts, getAccountsByBrand, getBrands } = require(
24
- path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'scripts', 'csv-reader')
25
- );
26
- const logger = require(
27
- path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'scripts', 'logger')
28
- );
29
-
30
- let accountsList;
31
- try {
32
- accountsList = loadAccounts();
33
- } catch (e) {
34
- // Fallback to JSON if CSV not found
35
- try {
36
- const jsonAccounts = JSON.parse(fs.readFileSync(path.join(configDir, 'accounts.json'), 'utf8'));
37
- accountsList = jsonAccounts.accounts.filter(a => a.active).map(a => ({
38
- account_id: a.id,
39
- brand: a.brand,
40
- platform: a.platforms ? a.platforms[0] : '',
41
- chrome_profile: a.chromeProfile,
42
- active: 'true',
43
- hasCredentials: false,
44
- }));
45
- // Expand multi-platform accounts
46
- const expanded = [];
47
- jsonAccounts.accounts.filter(a => a.active).forEach(a => {
48
- a.platforms.forEach(p => {
49
- expanded.push({
50
- account_id: `${a.id}-${p}`,
51
- brand: a.brand,
52
- brandSlug: a.brandSlug,
53
- platform: p,
54
- chrome_profile: a.chromeProfile,
55
- active: 'true',
56
- hasCredentials: false,
57
- });
58
- });
59
- });
60
- accountsList = expanded;
61
- } catch (e2) {
62
- console.error('❌ Cannot load accounts from CSV or JSON:', e.message);
63
- process.exit(1);
64
- }
65
- }
66
-
67
- const reportTypesConfig = JSON.parse(fs.readFileSync(path.join(configDir, 'report-types.json'), 'utf8'));
68
- const DEFAULT_OUTPUT_ROOT = path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'data');
69
-
70
- // --- CLI argument parsing ---
71
- const args = process.argv.slice(2);
72
- const getArg = (name) => {
73
- const idx = args.indexOf(`--${name}`);
74
- return idx !== -1 ? args[idx + 1] : null;
75
- };
76
- const hasFlag = (name) => args.includes(`--${name}`);
77
-
78
- // --- Commands ---
79
-
80
- if (hasFlag('list')) {
81
- // List all report types grouped by platform
82
- console.log('\n📋 Report Types Registry\n');
83
- const byPlatform = {};
84
- reportTypesConfig.reportTypes.forEach(rt => {
85
- if (!byPlatform[rt.platform]) byPlatform[rt.platform] = [];
86
- byPlatform[rt.platform].push(rt);
87
- });
88
-
89
- Object.entries(byPlatform).forEach(([platform, reports]) => {
90
- console.log(`\n🏪 ${platform.toUpperCase()} (${reports.length} types)`);
91
- console.log('─'.repeat(60));
92
- reports.forEach(r => {
93
- const status = r.status === 'active' ? '✅' : '🔄';
94
- const priority = r.priority ? '⭐' : ' ';
95
- console.log(` ${status} ${priority} ${r.id.padEnd(30)} ${r.frequency.padEnd(16)} ${r.exportFormat}`);
96
- });
97
- });
98
-
99
- console.log('\n\n👥 Accounts (from CSV):');
100
- const byBrand = {};
101
- accountsList.forEach(a => {
102
- if (!byBrand[a.brand]) byBrand[a.brand] = [];
103
- byBrand[a.brand].push(a.platform);
104
- });
105
- Object.entries(byBrand).forEach(([brand, platforms]) => {
106
- console.log(` • ${brand}: ${platforms.join(', ')} ${accountsList.find(a => a.brand === brand)?.hasCredentials ? '🔑' : '⚠️ no credentials'}`);
107
- });
108
-
109
- process.exit(0);
110
- }
111
-
112
- if (hasFlag('status')) {
113
- // Check download status for a period
114
- const outputRoot = args[args.indexOf('--status') + 1] || DEFAULT_OUTPUT_ROOT;
115
- const period = args[args.indexOf('--status') + 2] || getPreviousMonth();
116
-
117
- console.log(`\n📊 Download Status for ${period}\n`);
118
- console.log(`📂 Root: ${outputRoot}\n`);
119
-
120
- let found = 0;
121
- let missing = 0;
122
-
123
- const brandGroups = {};
124
- accountsList.forEach(a => {
125
- if (!brandGroups[a.brand]) brandGroups[a.brand] = { brand: a.brand, brandSlug: a.brand.replace(/ /g, '_'), platforms: [] };
126
- if (!brandGroups[a.brand].platforms.includes(a.platform)) brandGroups[a.brand].platforms.push(a.platform);
127
- });
128
- Object.values(brandGroups).forEach(account => {
129
- console.log(`\n👤 ${account.brand}`);
130
- account.platforms.forEach(platform => {
131
- const reports = getReportsForPlatform(platform);
132
- reports.forEach(rt => {
133
- const platformDir = platform.charAt(0).toUpperCase() + platform.slice(1);
134
- const filePath = path.join(
135
- outputRoot, platformDir, account.brandSlug, rt.id,
136
- `${period}_${account.brandSlug}_${rt.id}.${rt.fileExtension}`
137
- );
138
-
139
- if (fs.existsSync(filePath)) {
140
- const stats = fs.statSync(filePath);
141
- const sizeKb = (stats.size / 1024).toFixed(1);
142
- console.log(` ✅ ${platform}/${rt.id} (${sizeKb} KB)`);
143
- found++;
144
- } else {
145
- console.log(` ❌ ${platform}/${rt.id}`);
146
- missing++;
147
- }
148
- });
149
- });
150
- });
151
-
152
- console.log(`\n📊 Summary: ${found} found, ${missing} missing\n`);
153
- process.exit(0);
154
- }
155
-
156
- if (hasFlag('plan')) {
157
- // Generate a crawl plan for the browser agent
158
- const platform = getArg('platform');
159
- const brand = getArg('brand');
160
- const period = getArg('period') || getPreviousMonth();
161
- const outputRoot = getArg('output') || DEFAULT_OUTPUT_ROOT;
162
-
163
- console.log('\n🗺️ Crawl Plan\n');
164
- console.log(`Platform: ${platform || 'ALL'}`);
165
- console.log(`Brand: ${brand || 'ALL'}`);
166
- console.log(`Period: ${period}`);
167
- console.log(`Output: ${outputRoot}\n`);
168
-
169
- const tasks = generateCrawlTasks(platform, brand, period, outputRoot);
170
-
171
- tasks.forEach((task, i) => {
172
- console.log(`\n--- Task ${i + 1}/${tasks.length} ---`);
173
- console.log(`Platform: ${task.platform}`);
174
- console.log(`Brand: ${task.brand}`);
175
- console.log(`Report: ${task.reportType.name} (${task.reportType.id})`);
176
- console.log(`URL: ${task.reportType.url}`);
177
- console.log(`Save to: ${task.outputPath}`);
178
- console.log(`Steps:`);
179
- task.reportType.exportSteps.forEach((step, j) => {
180
- console.log(` ${j + 1}. ${step}`);
181
- });
182
- });
183
-
184
- console.log(`\n📋 Total tasks: ${tasks.length}`);
185
- process.exit(0);
186
- }
187
-
188
- if (hasFlag('dashboard')) {
189
- // Open dashboard in browser
190
- const dashboardPath = path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'dashboard', 'index.html');
191
- const { exec } = require('child_process');
192
- const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'start' : 'xdg-open';
193
- exec(`${cmd} "${dashboardPath}"`);
194
- console.log(`\n📊 Opening dashboard: ${dashboardPath}\n`);
195
- logger.activity({ action: 'open_dashboard', message: 'Dashboard opened' });
196
- process.exit(0);
197
- }
198
-
199
- // --- Default: show help ---
200
- console.log(`
201
- 🕷️ Marketplace Report Crawl Runner
202
- ====================================
203
-
204
- Commands:
205
- --list List all report types & accounts
206
- --status <output_root> <YYYYMM> Check download status
207
- --plan [options] Generate crawl plan for browser agent
208
- --dashboard Open dashboard in browser
209
-
210
- Plan options:
211
- --platform <shopee|lazada|tiktok> Filter by platform
212
- --brand <brand_name> Filter by brand
213
- --period <YYYYMM> Target period (default: previous month)
214
- --output <path> Output root directory
215
-
216
- Examples:
217
- node crawl-runner.js --list
218
- node crawl-runner.js --status ./data 202602
219
- node crawl-runner.js --plan --platform shopee --brand "Vita Dairy" --period 202602
220
- node crawl-runner.js --plan --platform lazada --period 202602
221
- node crawl-runner.js --dashboard
222
- `);
223
-
224
- // --- Helper functions ---
225
-
226
- function getPreviousMonth() {
227
- const d = new Date();
228
- d.setMonth(d.getMonth() - 1);
229
- return d.getFullYear().toString() + (d.getMonth() + 1).toString().padStart(2, '0');
230
- }
231
-
232
- function getReportsForPlatform(platform) {
233
- const seen = new Set();
234
- return reportTypesConfig.reportTypes.filter(rt => {
235
- if (rt.platform !== platform || rt.status === 'disabled') return false;
236
- if (seen.has(rt.id)) return false;
237
- seen.add(rt.id);
238
- return true;
239
- });
240
- }
241
-
242
- function generateCrawlTasks(platform, brand, period, outputRoot) {
243
- const tasks = [];
244
- const brandGroups = {};
245
- accountsList.forEach(a => {
246
- if (!brandGroups[a.brand]) brandGroups[a.brand] = { brand: a.brand, brandSlug: a.brand.replace(/ /g, '_'), platforms: [], chromeProfile: a.chrome_profile };
247
- if (!brandGroups[a.brand].platforms.includes(a.platform)) brandGroups[a.brand].platforms.push(a.platform);
248
- });
249
- const targetAccounts = Object.values(brandGroups).filter(a => {
250
- if (brand && a.brand !== brand) return false;
251
- return true;
252
- });
253
-
254
- targetAccounts.forEach(account => {
255
- const targetPlatforms = platform ? [platform] : account.platforms;
256
- targetPlatforms.forEach(p => {
257
- if (!account.platforms.includes(p)) return;
258
- const reports = getReportsForPlatform(p);
259
- reports.forEach(rt => {
260
- const platformDir = p.charAt(0).toUpperCase() + p.slice(1);
261
- const outputPath = path.join(
262
- outputRoot, platformDir, account.brandSlug, rt.id,
263
- `${period}_${account.brandSlug}_${rt.id}.${rt.fileExtension}`
264
- );
265
- tasks.push({
266
- platform: p,
267
- brand: account.brand,
268
- brandSlug: account.brandSlug,
269
- chromeProfile: account.chromeProfile,
270
- reportType: rt,
271
- period,
272
- outputPath
273
- });
274
- });
275
- });
276
- });
277
-
278
- return tasks;
279
- }
280
-
281
- module.exports = { generateCrawlTasks, getReportsForPlatform, getPreviousMonth };
@@ -1,72 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # Session Health Check — Validates if a marketplace session is still active
4
- # =============================================================================
5
- # Returns exit code 0 if session is valid, 1 if expired/invalid.
6
- # Designed to be called by other scripts or directly by AI agents.
7
- #
8
- # Usage:
9
- # ./session-check.sh <platform> <brand-id>
10
- # # Exit 0 = valid, Exit 1 = expired/no-profile
11
- # =============================================================================
12
-
13
- set -uo pipefail
14
-
15
- PROFILE_ROOT="${MARKETPLACE_PROFILE_ROOT:-$HOME/.marketplace-crawler/profiles}"
16
- AB_CMD="${AGENT_BROWSER_CMD:-npx -y agent-browser}"
17
-
18
- platform="${1:-}"
19
- brand_id="${2:-}"
20
-
21
- if [ -z "$platform" ] || [ -z "$brand_id" ]; then
22
- echo "Usage: $0 <platform> <brand-id>"
23
- echo "Example: $0 shopee vita-dairy"
24
- exit 2
25
- fi
26
-
27
- PROFILE_DIR="$PROFILE_ROOT/${platform}-${brand_id}"
28
-
29
- # Check if profile exists
30
- if [ ! -d "$PROFILE_DIR" ]; then
31
- echo "NO_PROFILE"
32
- exit 1
33
- fi
34
-
35
- # Define test URLs and login page indicators
36
- case "$platform" in
37
- shopee)
38
- CHECK_URL="https://banhang.shopee.vn/portal/dashboard/"
39
- LOGIN_PATTERN="buyer/login\|account/login"
40
- ;;
41
- lazada)
42
- CHECK_URL="https://sellercenter.lazada.vn/apps/dashboard"
43
- LOGIN_PATTERN="seller/login\|sso/login"
44
- ;;
45
- tiktok)
46
- CHECK_URL="https://seller-vn.tiktok.com/homepage?shop_region=VN"
47
- LOGIN_PATTERN="account/login\|passport"
48
- ;;
49
- *)
50
- echo "UNKNOWN_PLATFORM"
51
- exit 2
52
- ;;
53
- esac
54
-
55
- # Try to navigate to dashboard and check if redirected to login
56
- $AB_CMD --profile "$PROFILE_DIR" open "$CHECK_URL" 2>/dev/null
57
- sleep 3
58
- $AB_CMD --profile "$PROFILE_DIR" wait --load networkidle 2>/dev/null || true
59
-
60
- CURRENT_URL=$($AB_CMD --profile "$PROFILE_DIR" get url 2>/dev/null || echo "ERROR")
61
- $AB_CMD --profile "$PROFILE_DIR" close 2>/dev/null || true
62
-
63
- if echo "$CURRENT_URL" | grep -qi "$LOGIN_PATTERN"; then
64
- echo "EXPIRED"
65
- exit 1
66
- elif echo "$CURRENT_URL" | grep -qi "error\|ERROR"; then
67
- echo "ERROR"
68
- exit 1
69
- else
70
- echo "VALID"
71
- exit 0
72
- fi
@@ -1,349 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # Session Manager for Marketplace Report Crawler
4
- # =============================================================================
5
- # Cross-agent compatible: works from ANY AI coding agent that can run shell.
6
- # Uses agent-browser CLI for persistent browser sessions.
7
- #
8
- # Usage:
9
- # ./session-manager.sh status # Show all sessions
10
- # ./session-manager.sh login <platform> <brand-id> # Guided login (headed)
11
- # ./session-manager.sh login-all # Login all active accounts
12
- # ./session-manager.sh check <platform> <brand-id> # Check session health
13
- # ./session-manager.sh check-all # Check all sessions
14
- # =============================================================================
15
-
16
- set -euo pipefail
17
-
18
- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
19
- CONFIG_DIR="$SCRIPT_DIR/../config"
20
- ACCOUNTS_FILE="$CONFIG_DIR/accounts.json"
21
- SESSIONS_FILE="$CONFIG_DIR/sessions.json"
22
-
23
- # Default profile root — can be overridden via env
24
- PROFILE_ROOT="${MARKETPLACE_PROFILE_ROOT:-$HOME/.marketplace-crawler/profiles}"
25
-
26
- # agent-browser command — use npx if not installed globally
27
- AB_CMD="${AGENT_BROWSER_CMD:-npx -y agent-browser}"
28
-
29
- # Platform URLs via functions (compatible with bash 3.x on macOS)
30
- get_login_url() {
31
- case "$1" in
32
- shopee) echo "https://banhang.shopee.vn/" ;;
33
- lazada) echo "https://sellercenter.lazada.vn/" ;;
34
- tiktok) echo "https://seller-vn.tiktok.com/" ;;
35
- *) echo "" ;;
36
- esac
37
- }
38
-
39
- get_check_url() {
40
- case "$1" in
41
- shopee) echo "https://banhang.shopee.vn/portal/dashboard/" ;;
42
- lazada) echo "https://sellercenter.lazada.vn/apps/dashboard" ;;
43
- tiktok) echo "https://seller-vn.tiktok.com/homepage?shop_region=VN" ;;
44
- *) echo "" ;;
45
- esac
46
- }
47
-
48
- get_login_indicator() {
49
- case "$1" in
50
- shopee) echo "buyer/login" ;;
51
- lazada) echo "seller/login" ;;
52
- tiktok) echo "account/login" ;;
53
- *) echo "login" ;;
54
- esac
55
- }
56
-
57
- # --- Helpers ---
58
-
59
- get_profile_dir() {
60
- local platform="$1"
61
- local brand_id="$2"
62
- echo "$PROFILE_ROOT/${platform}-${brand_id}"
63
- }
64
-
65
- ensure_profile_dir() {
66
- local dir="$1"
67
- mkdir -p "$dir"
68
- }
69
-
70
- get_brands() {
71
- # Returns brand IDs from accounts.json
72
- node -e "
73
- const a = require('$ACCOUNTS_FILE');
74
- a.accounts.filter(x => x.active).forEach(x => console.log(x.id));
75
- "
76
- }
77
-
78
- get_brand_platforms() {
79
- local brand_id="$1"
80
- node -e "
81
- const a = require('$ACCOUNTS_FILE');
82
- const b = a.accounts.find(x => x.id === '$brand_id');
83
- if (b) b.platforms.forEach(p => console.log(p));
84
- "
85
- }
86
-
87
- get_brand_name() {
88
- local brand_id="$1"
89
- node -e "
90
- const a = require('$ACCOUNTS_FILE');
91
- const b = a.accounts.find(x => x.id === '$brand_id');
92
- if (b) console.log(b.brand);
93
- "
94
- }
95
-
96
- update_session_status() {
97
- local platform="$1"
98
- local brand_id="$2"
99
- local status="$3"
100
- local now
101
- now=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
102
-
103
- # Create sessions.json if not exists
104
- if [ ! -f "$SESSIONS_FILE" ]; then
105
- echo '{"sessions":[]}' > "$SESSIONS_FILE"
106
- fi
107
-
108
- node -e "
109
- const fs = require('fs');
110
- const data = JSON.parse(fs.readFileSync('$SESSIONS_FILE', 'utf8'));
111
- const idx = data.sessions.findIndex(s => s.platform === '$platform' && s.brandId === '$brand_id');
112
- const entry = {
113
- platform: '$platform',
114
- brandId: '$brand_id',
115
- status: '$status',
116
- profilePath: '$(get_profile_dir "$platform" "$brand_id")',
117
- $([ "$status" = "valid" ] && echo "lastLogin: '$now'," || echo "")
118
- lastCheck: '$now'
119
- };
120
- if (idx >= 0) { Object.assign(data.sessions[idx], entry); }
121
- else { data.sessions.push(entry); }
122
- fs.writeFileSync('$SESSIONS_FILE', JSON.stringify(data, null, 2));
123
- "
124
- }
125
-
126
- # --- Commands ---
127
-
128
- cmd_status() {
129
- echo ""
130
- echo "📊 Marketplace Session Status"
131
- echo "═══════════════════════════════════════════════════════"
132
- echo ""
133
- echo "Profile root: $PROFILE_ROOT"
134
- echo ""
135
-
136
- if [ ! -f "$SESSIONS_FILE" ]; then
137
- echo "⚠️ No sessions tracked yet. Run 'login-all' first."
138
- return
139
- fi
140
-
141
- node -e "
142
- const fs = require('fs');
143
- const data = JSON.parse(fs.readFileSync('$SESSIONS_FILE', 'utf8'));
144
- if (data.sessions.length === 0) {
145
- console.log('⚠️ No sessions tracked yet.');
146
- process.exit(0);
147
- }
148
- const icons = { valid: '✅', expired: '❌', unknown: '❓' };
149
- data.sessions.forEach(s => {
150
- const icon = icons[s.status] || '❓';
151
- const last = s.lastCheck ? new Date(s.lastCheck).toLocaleString() : 'never';
152
- console.log(\` \${icon} \${s.platform.padEnd(8)} \${s.brandId.padEnd(15)} \${s.status.padEnd(10)} last: \${last}\`);
153
- });
154
- "
155
- echo ""
156
- }
157
-
158
- cmd_login() {
159
- local platform="$1"
160
- local brand_id="$2"
161
- local brand_name
162
- brand_name=$(get_brand_name "$brand_id")
163
- local profile_dir
164
- profile_dir=$(get_profile_dir "$platform" "$brand_id")
165
- local login_url
166
- login_url=$(get_login_url "$platform")
167
-
168
- ensure_profile_dir "$profile_dir"
169
-
170
- echo ""
171
- echo "🔐 Guided Login: $brand_name on ${platform^^}"
172
- echo "═══════════════════════════════════════════════════════"
173
- echo ""
174
- echo "📂 Profile: $profile_dir"
175
- echo "🌐 URL: $login_url"
176
- echo ""
177
- echo "⏳ Opening browser in headed mode..."
178
- echo " → Please login manually (solve CAPTCHA/OTP as needed)"
179
- echo " → When you see the dashboard, press Enter here to save session"
180
- echo ""
181
-
182
- # Open browser in headed mode with persistent profile
183
- $AB_CMD --profile "$profile_dir" --headed open "$login_url" &
184
- local ab_pid=$!
185
-
186
- # Wait for user to confirm login
187
- read -r -p "✅ Press Enter when login is complete... "
188
-
189
- # Take a screenshot as proof
190
- local screenshot_dir="$profile_dir/screenshots"
191
- mkdir -p "$screenshot_dir"
192
- $AB_CMD --profile "$profile_dir" screenshot "$screenshot_dir/login-success-$(date +%Y%m%d).png" 2>/dev/null || true
193
-
194
- # Close the browser (profile is auto-saved)
195
- $AB_CMD --profile "$profile_dir" close 2>/dev/null || true
196
-
197
- # Update session tracking
198
- update_session_status "$platform" "$brand_id" "valid"
199
-
200
- echo ""
201
- echo "✅ Session saved for $brand_name on ${platform^^}"
202
- echo " Profile: $profile_dir"
203
- echo ""
204
- }
205
-
206
- cmd_login_all() {
207
- echo ""
208
- echo "🔐 Guided Login — All Active Accounts"
209
- echo "═══════════════════════════════════════════════════════"
210
- echo ""
211
-
212
- local brands
213
- brands=$(get_brands)
214
-
215
- for brand_id in $brands; do
216
- local platforms
217
- platforms=$(get_brand_platforms "$brand_id")
218
- for platform in $platforms; do
219
- cmd_login "$platform" "$brand_id"
220
- done
221
- done
222
-
223
- echo ""
224
- echo "✅ All accounts logged in. Run 'check-all' to verify."
225
- echo ""
226
- }
227
-
228
- cmd_check() {
229
- local platform="$1"
230
- local brand_id="$2"
231
- local brand_name
232
- brand_name=$(get_brand_name "$brand_id")
233
- local profile_dir
234
- profile_dir=$(get_profile_dir "$platform" "$brand_id")
235
- local check_url
236
- check_url=$(get_check_url "$platform")
237
- local login_indicator
238
- login_indicator=$(get_login_indicator "$platform")
239
-
240
- if [ ! -d "$profile_dir" ]; then
241
- echo "❌ ${platform}/${brand_id}: No profile found (never logged in)"
242
- update_session_status "$platform" "$brand_id" "unknown"
243
- return 1
244
- fi
245
-
246
- # Open in headless mode, navigate to dashboard, check URL
247
- $AB_CMD --profile "$profile_dir" open "$check_url" 2>/dev/null || true
248
- sleep 3
249
- $AB_CMD --profile "$profile_dir" wait --load networkidle 2>/dev/null || true
250
-
251
- local current_url
252
- current_url=$($AB_CMD --profile "$profile_dir" get url 2>/dev/null || echo "error")
253
-
254
- $AB_CMD --profile "$profile_dir" close 2>/dev/null || true
255
-
256
- if echo "$current_url" | grep -qi "$login_indicator"; then
257
- echo "❌ ${platform}/${brand_id} ($brand_name): Session EXPIRED"
258
- update_session_status "$platform" "$brand_id" "expired"
259
- return 1
260
- else
261
- echo "✅ ${platform}/${brand_id} ($brand_name): Session VALID"
262
- update_session_status "$platform" "$brand_id" "valid"
263
- return 0
264
- fi
265
- }
266
-
267
- cmd_check_all() {
268
- echo ""
269
- echo "🔍 Checking All Sessions"
270
- echo "═══════════════════════════════════════════════════════"
271
- echo ""
272
-
273
- local valid=0
274
- local expired=0
275
- local unknown=0
276
-
277
- local brands
278
- brands=$(get_brands)
279
-
280
- for brand_id in $brands; do
281
- local platforms
282
- platforms=$(get_brand_platforms "$brand_id")
283
- for platform in $platforms; do
284
- if cmd_check "$platform" "$brand_id"; then
285
- ((valid++))
286
- else
287
- ((expired++))
288
- fi
289
- done
290
- done
291
-
292
- echo ""
293
- echo "📊 Summary: $valid valid, $expired expired/unknown"
294
- echo ""
295
- }
296
-
297
- # --- Main ---
298
-
299
- case "${1:-help}" in
300
- status)
301
- cmd_status
302
- ;;
303
- login)
304
- if [ $# -lt 3 ]; then
305
- echo "Usage: $0 login <platform> <brand-id>"
306
- echo "Platforms: shopee, lazada, tiktok"
307
- exit 1
308
- fi
309
- cmd_login "$2" "$3"
310
- ;;
311
- login-all)
312
- cmd_login_all
313
- ;;
314
- check)
315
- if [ $# -lt 3 ]; then
316
- echo "Usage: $0 check <platform> <brand-id>"
317
- exit 1
318
- fi
319
- cmd_check "$2" "$3"
320
- ;;
321
- check-all)
322
- cmd_check_all
323
- ;;
324
- *)
325
- echo ""
326
- echo "🕷️ Marketplace Session Manager"
327
- echo "════════════════════════════════"
328
- echo ""
329
- echo "Commands:"
330
- echo " status Show all session statuses"
331
- echo " login <platform> <brand-id> Guided login (opens browser)"
332
- echo " login-all Login all active accounts"
333
- echo " check <platform> <brand-id> Check if session is valid"
334
- echo " check-all Check all sessions"
335
- echo ""
336
- echo "Platforms: shopee, lazada, tiktok"
337
- echo "Brands: $(get_brands | tr '\n' ', ')"
338
- echo ""
339
- echo "Environment:"
340
- echo " MARKETPLACE_PROFILE_ROOT Profile directory (default: ~/.marketplace-crawler/profiles)"
341
- echo " AGENT_BROWSER_CMD agent-browser command (default: npx -y agent-browser)"
342
- echo ""
343
- echo "Examples:"
344
- echo " $0 login shopee vita-dairy"
345
- echo " $0 check-all"
346
- echo " $0 status"
347
- echo ""
348
- ;;
349
- esac