codymaster 4.4.5 → 4.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +29 -14
  3. package/commands/demo.md +1 -1
  4. package/dist/context-bus.js +70 -0
  5. package/dist/context-db.js +265 -0
  6. package/dist/continuity.js +12 -0
  7. package/dist/file-watcher.js +79 -0
  8. package/dist/index.js +152 -1
  9. package/dist/l0-indexer.js +158 -0
  10. package/dist/mcp-context-server.js +400 -0
  11. package/dist/migrate-json-to-sqlite.js +126 -0
  12. package/dist/skill-chain.js +19 -3
  13. package/dist/token-budget.js +108 -0
  14. package/dist/uri-resolver.js +203 -0
  15. package/package.json +4 -1
  16. package/skills/_shared/helpers.md +50 -14
  17. package/skills/cm-autopilot/SKILL.md +29 -0
  18. package/skills/cm-autopilot/scripts/autopilot.py +190 -0
  19. package/skills/cm-continuity/SKILL.md +90 -28
  20. package/skills/cm-skill-chain/SKILL.md +47 -1
  21. package/skills/cm-start/SKILL.md +11 -2
  22. package/skills/boxme-git-config/SKILL.md +0 -56
  23. package/skills/boxme-local-dev/SKILL.md +0 -66
  24. package/skills/jobs-to-be-done/SKILL.md +0 -266
  25. package/skills/jobs-to-be-done/references/case-studies.md +0 -154
  26. package/skills/jobs-to-be-done/references/competitive-strategy.md +0 -280
  27. package/skills/jobs-to-be-done/references/diagnostics.md +0 -158
  28. package/skills/jobs-to-be-done/references/innovation-process.md +0 -392
  29. package/skills/jobs-to-be-done/references/organizational-change.md +0 -328
  30. package/skills/marketplace-report-crawler/SKILL.md +0 -176
  31. package/skills/marketplace-report-crawler/config/accounts.json +0 -41
  32. package/skills/marketplace-report-crawler/config/report-types.json +0 -422
  33. package/skills/marketplace-report-crawler/config/sessions.json +0 -3
  34. package/skills/marketplace-report-crawler/scripts/ab-wrapper.sh +0 -102
  35. package/skills/marketplace-report-crawler/scripts/browser-actions/lazada/lazada-actions.js +0 -114
  36. package/skills/marketplace-report-crawler/scripts/browser-actions/shopee/shopee-actions.js +0 -94
  37. package/skills/marketplace-report-crawler/scripts/browser-actions/tiktok/tiktok-actions.js +0 -272
  38. package/skills/marketplace-report-crawler/scripts/crawl-runner.js +0 -281
  39. package/skills/marketplace-report-crawler/scripts/session-check.sh +0 -72
  40. package/skills/marketplace-report-crawler/scripts/session-manager.sh +0 -349
  41. package/skills/marketplace-report-crawler/scripts/setup-folders.sh +0 -83
  42. package/skills/medical-research/SKILL.md +0 -194
  43. package/skills/medical-research/scripts/evidence_checker.py +0 -288
  44. package/skills/mom-test/SKILL.md +0 -267
  45. package/skills/mom-test/references/avoiding-bad-data.md +0 -221
  46. package/skills/mom-test/references/case-studies.md +0 -306
  47. package/skills/mom-test/references/commitment-advancement.md +0 -219
  48. package/skills/mom-test/references/finding-conversations.md +0 -251
  49. package/skills/mom-test/references/processing-learning.md +0 -256
  50. package/skills/mom-test/references/question-patterns.md +0 -198
  51. package/skills/pandasai-analytics/SKILL.md +0 -251
  52. package/skills/release-it/SKILL.md +0 -235
  53. package/skills/release-it/references/anti-patterns.md +0 -279
  54. package/skills/release-it/references/capacity-planning.md +0 -285
  55. package/skills/release-it/references/chaos-engineering.md +0 -325
  56. package/skills/release-it/references/deployment-strategies.md +0 -331
  57. package/skills/release-it/references/observability.md +0 -301
  58. package/skills/release-it/references/stability-patterns.md +0 -355
  59. package/skills/skill-creator-ultra/.agents/workflows/skill-audit.md +0 -37
  60. package/skills/skill-creator-ultra/.agents/workflows/skill-compare.md +0 -34
  61. package/skills/skill-creator-ultra/.agents/workflows/skill-export.md +0 -51
  62. package/skills/skill-creator-ultra/.agents/workflows/skill-generate.md +0 -39
  63. package/skills/skill-creator-ultra/.agents/workflows/skill-scaffold.md +0 -52
  64. package/skills/skill-creator-ultra/.agents/workflows/skill-simulate.md +0 -25
  65. package/skills/skill-creator-ultra/.agents/workflows/skill-stats.md +0 -31
  66. package/skills/skill-creator-ultra/.agents/workflows/skill-validate.md +0 -25
  67. package/skills/skill-creator-ultra/README.md +0 -1242
  68. package/skills/skill-creator-ultra/SKILL.md +0 -388
  69. package/skills/skill-creator-ultra/agents/analyzer.md +0 -274
  70. package/skills/skill-creator-ultra/agents/comparator.md +0 -202
  71. package/skills/skill-creator-ultra/agents/grader.md +0 -223
  72. package/skills/skill-creator-ultra/assets/eval_review.html +0 -146
  73. package/skills/skill-creator-ultra/eval-viewer/generate_review.py +0 -471
  74. package/skills/skill-creator-ultra/eval-viewer/viewer.html +0 -1325
  75. package/skills/skill-creator-ultra/examples/example_anthropic_frontend.md +0 -109
  76. package/skills/skill-creator-ultra/examples/example_anthropic_pdf.md +0 -116
  77. package/skills/skill-creator-ultra/examples/example_api_docs.md +0 -189
  78. package/skills/skill-creator-ultra/examples/example_db_migration.md +0 -253
  79. package/skills/skill-creator-ultra/examples/example_git_commit.md +0 -111
  80. package/skills/skill-creator-ultra/install.ps1 +0 -289
  81. package/skills/skill-creator-ultra/install.sh +0 -313
  82. package/skills/skill-creator-ultra/phases/phase1_interview.md +0 -202
  83. package/skills/skill-creator-ultra/phases/phase2_extract.md +0 -55
  84. package/skills/skill-creator-ultra/phases/phase3_detect.md +0 -57
  85. package/skills/skill-creator-ultra/phases/phase4_generate.md +0 -543
  86. package/skills/skill-creator-ultra/phases/phase5_test.md +0 -319
  87. package/skills/skill-creator-ultra/phases/phase6_eval.md +0 -301
  88. package/skills/skill-creator-ultra/phases/phase7_iterate.md +0 -103
  89. package/skills/skill-creator-ultra/phases/phase8_optimize.md +0 -113
  90. package/skills/skill-creator-ultra/resources/advanced_patterns.md +0 -499
  91. package/skills/skill-creator-ultra/resources/anti_patterns.md +0 -376
  92. package/skills/skill-creator-ultra/resources/blueprints.md +0 -498
  93. package/skills/skill-creator-ultra/resources/checklist.md +0 -243
  94. package/skills/skill-creator-ultra/resources/composition_cookbook.md +0 -291
  95. package/skills/skill-creator-ultra/resources/description_optimization.md +0 -90
  96. package/skills/skill-creator-ultra/resources/eval_guide.md +0 -133
  97. package/skills/skill-creator-ultra/resources/industry_questions.md +0 -189
  98. package/skills/skill-creator-ultra/resources/interview_questions.md +0 -200
  99. package/skills/skill-creator-ultra/resources/pattern_detection.md +0 -200
  100. package/skills/skill-creator-ultra/resources/prompt_engineering.md +0 -531
  101. package/skills/skill-creator-ultra/resources/schemas.md +0 -430
  102. package/skills/skill-creator-ultra/resources/script_integration.md +0 -593
  103. package/skills/skill-creator-ultra/resources/scripts_guide.md +0 -339
  104. package/skills/skill-creator-ultra/resources/skill_template.md +0 -124
  105. package/skills/skill-creator-ultra/resources/skill_writing_guide.md +0 -634
  106. package/skills/skill-creator-ultra/resources/versioning_guide.md +0 -193
  107. package/skills/skill-creator-ultra/scripts/ci_eval.py +0 -200
  108. package/skills/skill-creator-ultra/scripts/package_skill.py +0 -165
  109. package/skills/skill-creator-ultra/scripts/simulate_skill.py +0 -398
  110. package/skills/skill-creator-ultra/scripts/skill_audit.py +0 -611
  111. package/skills/skill-creator-ultra/scripts/skill_compare.py +0 -265
  112. package/skills/skill-creator-ultra/scripts/skill_export.py +0 -334
  113. package/skills/skill-creator-ultra/scripts/skill_scaffold.py +0 -403
  114. package/skills/skill-creator-ultra/scripts/skill_stats.py +0 -339
  115. package/skills/skill-creator-ultra/scripts/validate_skill.py +0 -411
  116. package/skills/tailwind-mastery/SKILL.md +0 -229
  117. package/skills/vercel-react-best-practices/AGENTS.md +0 -3373
  118. package/skills/vercel-react-best-practices/README.md +0 -123
  119. package/skills/vercel-react-best-practices/SKILL.md +0 -143
  120. package/skills/vercel-react-best-practices/rules/_sections.md +0 -46
  121. package/skills/vercel-react-best-practices/rules/_template.md +0 -28
  122. package/skills/vercel-react-best-practices/rules/advanced-event-handler-refs.md +0 -55
  123. package/skills/vercel-react-best-practices/rules/advanced-init-once.md +0 -42
  124. package/skills/vercel-react-best-practices/rules/advanced-use-latest.md +0 -39
  125. package/skills/vercel-react-best-practices/rules/async-api-routes.md +0 -38
  126. package/skills/vercel-react-best-practices/rules/async-defer-await.md +0 -80
  127. package/skills/vercel-react-best-practices/rules/async-dependencies.md +0 -51
  128. package/skills/vercel-react-best-practices/rules/async-parallel.md +0 -28
  129. package/skills/vercel-react-best-practices/rules/async-suspense-boundaries.md +0 -99
  130. package/skills/vercel-react-best-practices/rules/bundle-barrel-imports.md +0 -59
  131. package/skills/vercel-react-best-practices/rules/bundle-conditional.md +0 -31
  132. package/skills/vercel-react-best-practices/rules/bundle-defer-third-party.md +0 -49
  133. package/skills/vercel-react-best-practices/rules/bundle-dynamic-imports.md +0 -35
  134. package/skills/vercel-react-best-practices/rules/bundle-preload.md +0 -50
  135. package/skills/vercel-react-best-practices/rules/client-event-listeners.md +0 -74
  136. package/skills/vercel-react-best-practices/rules/client-localstorage-schema.md +0 -71
  137. package/skills/vercel-react-best-practices/rules/client-passive-event-listeners.md +0 -48
  138. package/skills/vercel-react-best-practices/rules/client-swr-dedup.md +0 -56
  139. package/skills/vercel-react-best-practices/rules/js-batch-dom-css.md +0 -107
  140. package/skills/vercel-react-best-practices/rules/js-cache-function-results.md +0 -80
  141. package/skills/vercel-react-best-practices/rules/js-cache-property-access.md +0 -28
  142. package/skills/vercel-react-best-practices/rules/js-cache-storage.md +0 -70
  143. package/skills/vercel-react-best-practices/rules/js-combine-iterations.md +0 -32
  144. package/skills/vercel-react-best-practices/rules/js-early-exit.md +0 -50
  145. package/skills/vercel-react-best-practices/rules/js-flatmap-filter.md +0 -60
  146. package/skills/vercel-react-best-practices/rules/js-hoist-regexp.md +0 -45
  147. package/skills/vercel-react-best-practices/rules/js-index-maps.md +0 -37
  148. package/skills/vercel-react-best-practices/rules/js-length-check-first.md +0 -49
  149. package/skills/vercel-react-best-practices/rules/js-min-max-loop.md +0 -82
  150. package/skills/vercel-react-best-practices/rules/js-set-map-lookups.md +0 -24
  151. package/skills/vercel-react-best-practices/rules/js-tosorted-immutable.md +0 -57
  152. package/skills/vercel-react-best-practices/rules/rendering-activity.md +0 -26
  153. package/skills/vercel-react-best-practices/rules/rendering-animate-svg-wrapper.md +0 -47
  154. package/skills/vercel-react-best-practices/rules/rendering-conditional-render.md +0 -40
  155. package/skills/vercel-react-best-practices/rules/rendering-content-visibility.md +0 -38
  156. package/skills/vercel-react-best-practices/rules/rendering-hoist-jsx.md +0 -46
  157. package/skills/vercel-react-best-practices/rules/rendering-hydration-no-flicker.md +0 -82
  158. package/skills/vercel-react-best-practices/rules/rendering-hydration-suppress-warning.md +0 -30
  159. package/skills/vercel-react-best-practices/rules/rendering-resource-hints.md +0 -85
  160. package/skills/vercel-react-best-practices/rules/rendering-script-defer-async.md +0 -68
  161. package/skills/vercel-react-best-practices/rules/rendering-svg-precision.md +0 -28
  162. package/skills/vercel-react-best-practices/rules/rendering-usetransition-loading.md +0 -75
  163. package/skills/vercel-react-best-practices/rules/rerender-defer-reads.md +0 -39
  164. package/skills/vercel-react-best-practices/rules/rerender-dependencies.md +0 -45
  165. package/skills/vercel-react-best-practices/rules/rerender-derived-state-no-effect.md +0 -40
  166. package/skills/vercel-react-best-practices/rules/rerender-derived-state.md +0 -29
  167. package/skills/vercel-react-best-practices/rules/rerender-functional-setstate.md +0 -74
  168. package/skills/vercel-react-best-practices/rules/rerender-lazy-state-init.md +0 -58
  169. package/skills/vercel-react-best-practices/rules/rerender-memo-with-default-value.md +0 -38
  170. package/skills/vercel-react-best-practices/rules/rerender-memo.md +0 -44
  171. package/skills/vercel-react-best-practices/rules/rerender-move-effect-to-event.md +0 -45
  172. package/skills/vercel-react-best-practices/rules/rerender-no-inline-components.md +0 -82
  173. package/skills/vercel-react-best-practices/rules/rerender-simple-expression-in-memo.md +0 -35
  174. package/skills/vercel-react-best-practices/rules/rerender-split-combined-hooks.md +0 -64
  175. package/skills/vercel-react-best-practices/rules/rerender-transitions.md +0 -40
  176. package/skills/vercel-react-best-practices/rules/rerender-use-deferred-value.md +0 -59
  177. package/skills/vercel-react-best-practices/rules/rerender-use-ref-transient-values.md +0 -73
  178. package/skills/vercel-react-best-practices/rules/server-after-nonblocking.md +0 -73
  179. package/skills/vercel-react-best-practices/rules/server-auth-actions.md +0 -96
  180. package/skills/vercel-react-best-practices/rules/server-cache-lru.md +0 -41
  181. package/skills/vercel-react-best-practices/rules/server-cache-react.md +0 -76
  182. package/skills/vercel-react-best-practices/rules/server-dedup-props.md +0 -65
  183. package/skills/vercel-react-best-practices/rules/server-hoist-static-io.md +0 -142
  184. package/skills/vercel-react-best-practices/rules/server-parallel-fetching.md +0 -83
  185. package/skills/vercel-react-best-practices/rules/server-serialization.md +0 -38
  186. package/skills/web-design-guidelines/SKILL.md +0 -39
@@ -1,281 +0,0 @@
1
- /**
2
- * Marketplace Report Crawl Runner
3
- * ================================
4
- * Orchestrator that reads config files and generates crawl instructions
5
- * for the browser agent to execute.
6
- *
7
- * UPDATED: Now reads from accounts.csv (via csv-reader.js) instead of accounts.json
8
- * Includes logging integration for activity & bug tracking.
9
- *
10
- * Usage:
11
- * node crawl-runner.js --platform shopee --brand "Vita Dairy" --period 202602
12
- * node crawl-runner.js --list # List all report types
13
- * node crawl-runner.js --setup-folders ./data # Generate folder structure
14
- * node crawl-runner.js --status ./data 202602 # Check download status
15
- * node crawl-runner.js --dashboard # Open dashboard
16
- */
17
-
18
- const fs = require('fs');
19
- const path = require('path');
20
-
21
- // --- Load configs ---
22
- const configDir = path.join(__dirname, '..', 'config');
23
- const { loadAccounts, getAccountsByBrand, getBrands } = require(
24
- path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'scripts', 'csv-reader')
25
- );
26
- const logger = require(
27
- path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'scripts', 'logger')
28
- );
29
-
30
- let accountsList;
31
- try {
32
- accountsList = loadAccounts();
33
- } catch (e) {
34
- // Fallback to JSON if CSV not found
35
- try {
36
- const jsonAccounts = JSON.parse(fs.readFileSync(path.join(configDir, 'accounts.json'), 'utf8'));
37
- accountsList = jsonAccounts.accounts.filter(a => a.active).map(a => ({
38
- account_id: a.id,
39
- brand: a.brand,
40
- platform: a.platforms ? a.platforms[0] : '',
41
- chrome_profile: a.chromeProfile,
42
- active: 'true',
43
- hasCredentials: false,
44
- }));
45
- // Expand multi-platform accounts
46
- const expanded = [];
47
- jsonAccounts.accounts.filter(a => a.active).forEach(a => {
48
- a.platforms.forEach(p => {
49
- expanded.push({
50
- account_id: `${a.id}-${p}`,
51
- brand: a.brand,
52
- brandSlug: a.brandSlug,
53
- platform: p,
54
- chrome_profile: a.chromeProfile,
55
- active: 'true',
56
- hasCredentials: false,
57
- });
58
- });
59
- });
60
- accountsList = expanded;
61
- } catch (e2) {
62
- console.error('❌ Cannot load accounts from CSV or JSON:', e.message);
63
- process.exit(1);
64
- }
65
- }
66
-
67
- const reportTypesConfig = JSON.parse(fs.readFileSync(path.join(configDir, 'report-types.json'), 'utf8'));
68
- const DEFAULT_OUTPUT_ROOT = path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'data');
69
-
70
- // --- CLI argument parsing ---
71
- const args = process.argv.slice(2);
72
- const getArg = (name) => {
73
- const idx = args.indexOf(`--${name}`);
74
- return idx !== -1 ? args[idx + 1] : null;
75
- };
76
- const hasFlag = (name) => args.includes(`--${name}`);
77
-
78
- // --- Commands ---
79
-
80
- if (hasFlag('list')) {
81
- // List all report types grouped by platform
82
- console.log('\n📋 Report Types Registry\n');
83
- const byPlatform = {};
84
- reportTypesConfig.reportTypes.forEach(rt => {
85
- if (!byPlatform[rt.platform]) byPlatform[rt.platform] = [];
86
- byPlatform[rt.platform].push(rt);
87
- });
88
-
89
- Object.entries(byPlatform).forEach(([platform, reports]) => {
90
- console.log(`\n🏪 ${platform.toUpperCase()} (${reports.length} types)`);
91
- console.log('─'.repeat(60));
92
- reports.forEach(r => {
93
- const status = r.status === 'active' ? '✅' : '🔄';
94
- const priority = r.priority ? '⭐' : ' ';
95
- console.log(` ${status} ${priority} ${r.id.padEnd(30)} ${r.frequency.padEnd(16)} ${r.exportFormat}`);
96
- });
97
- });
98
-
99
- console.log('\n\n👥 Accounts (from CSV):');
100
- const byBrand = {};
101
- accountsList.forEach(a => {
102
- if (!byBrand[a.brand]) byBrand[a.brand] = [];
103
- byBrand[a.brand].push(a.platform);
104
- });
105
- Object.entries(byBrand).forEach(([brand, platforms]) => {
106
- console.log(` • ${brand}: ${platforms.join(', ')} ${accountsList.find(a => a.brand === brand)?.hasCredentials ? '🔑' : '⚠️ no credentials'}`);
107
- });
108
-
109
- process.exit(0);
110
- }
111
-
112
- if (hasFlag('status')) {
113
- // Check download status for a period
114
- const outputRoot = args[args.indexOf('--status') + 1] || DEFAULT_OUTPUT_ROOT;
115
- const period = args[args.indexOf('--status') + 2] || getPreviousMonth();
116
-
117
- console.log(`\n📊 Download Status for ${period}\n`);
118
- console.log(`📂 Root: ${outputRoot}\n`);
119
-
120
- let found = 0;
121
- let missing = 0;
122
-
123
- const brandGroups = {};
124
- accountsList.forEach(a => {
125
- if (!brandGroups[a.brand]) brandGroups[a.brand] = { brand: a.brand, brandSlug: a.brand.replace(/ /g, '_'), platforms: [] };
126
- if (!brandGroups[a.brand].platforms.includes(a.platform)) brandGroups[a.brand].platforms.push(a.platform);
127
- });
128
- Object.values(brandGroups).forEach(account => {
129
- console.log(`\n👤 ${account.brand}`);
130
- account.platforms.forEach(platform => {
131
- const reports = getReportsForPlatform(platform);
132
- reports.forEach(rt => {
133
- const platformDir = platform.charAt(0).toUpperCase() + platform.slice(1);
134
- const filePath = path.join(
135
- outputRoot, platformDir, account.brandSlug, rt.id,
136
- `${period}_${account.brandSlug}_${rt.id}.${rt.fileExtension}`
137
- );
138
-
139
- if (fs.existsSync(filePath)) {
140
- const stats = fs.statSync(filePath);
141
- const sizeKb = (stats.size / 1024).toFixed(1);
142
- console.log(` ✅ ${platform}/${rt.id} (${sizeKb} KB)`);
143
- found++;
144
- } else {
145
- console.log(` ❌ ${platform}/${rt.id}`);
146
- missing++;
147
- }
148
- });
149
- });
150
- });
151
-
152
- console.log(`\n📊 Summary: ${found} found, ${missing} missing\n`);
153
- process.exit(0);
154
- }
155
-
156
- if (hasFlag('plan')) {
157
- // Generate a crawl plan for the browser agent
158
- const platform = getArg('platform');
159
- const brand = getArg('brand');
160
- const period = getArg('period') || getPreviousMonth();
161
- const outputRoot = getArg('output') || DEFAULT_OUTPUT_ROOT;
162
-
163
- console.log('\n🗺️ Crawl Plan\n');
164
- console.log(`Platform: ${platform || 'ALL'}`);
165
- console.log(`Brand: ${brand || 'ALL'}`);
166
- console.log(`Period: ${period}`);
167
- console.log(`Output: ${outputRoot}\n`);
168
-
169
- const tasks = generateCrawlTasks(platform, brand, period, outputRoot);
170
-
171
- tasks.forEach((task, i) => {
172
- console.log(`\n--- Task ${i + 1}/${tasks.length} ---`);
173
- console.log(`Platform: ${task.platform}`);
174
- console.log(`Brand: ${task.brand}`);
175
- console.log(`Report: ${task.reportType.name} (${task.reportType.id})`);
176
- console.log(`URL: ${task.reportType.url}`);
177
- console.log(`Save to: ${task.outputPath}`);
178
- console.log(`Steps:`);
179
- task.reportType.exportSteps.forEach((step, j) => {
180
- console.log(` ${j + 1}. ${step}`);
181
- });
182
- });
183
-
184
- console.log(`\n📋 Total tasks: ${tasks.length}`);
185
- process.exit(0);
186
- }
187
-
188
- if (hasFlag('dashboard')) {
189
- // Open dashboard in browser
190
- const dashboardPath = path.join(__dirname, '..', '..', '..', 'Marketplace_Report', 'dashboard', 'index.html');
191
- const { exec } = require('child_process');
192
- const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'start' : 'xdg-open';
193
- exec(`${cmd} "${dashboardPath}"`);
194
- console.log(`\n📊 Opening dashboard: ${dashboardPath}\n`);
195
- logger.activity({ action: 'open_dashboard', message: 'Dashboard opened' });
196
- process.exit(0);
197
- }
198
-
199
- // --- Default: show help ---
200
- console.log(`
201
- 🕷️ Marketplace Report Crawl Runner
202
- ====================================
203
-
204
- Commands:
205
- --list List all report types & accounts
206
- --status <output_root> <YYYYMM> Check download status
207
- --plan [options] Generate crawl plan for browser agent
208
- --dashboard Open dashboard in browser
209
-
210
- Plan options:
211
- --platform <shopee|lazada|tiktok> Filter by platform
212
- --brand <brand_name> Filter by brand
213
- --period <YYYYMM> Target period (default: previous month)
214
- --output <path> Output root directory
215
-
216
- Examples:
217
- node crawl-runner.js --list
218
- node crawl-runner.js --status ./data 202602
219
- node crawl-runner.js --plan --platform shopee --brand "Vita Dairy" --period 202602
220
- node crawl-runner.js --plan --platform lazada --period 202602
221
- node crawl-runner.js --dashboard
222
- `);
223
-
224
- // --- Helper functions ---
225
-
226
- function getPreviousMonth() {
227
- const d = new Date();
228
- d.setMonth(d.getMonth() - 1);
229
- return d.getFullYear().toString() + (d.getMonth() + 1).toString().padStart(2, '0');
230
- }
231
-
232
- function getReportsForPlatform(platform) {
233
- const seen = new Set();
234
- return reportTypesConfig.reportTypes.filter(rt => {
235
- if (rt.platform !== platform || rt.status === 'disabled') return false;
236
- if (seen.has(rt.id)) return false;
237
- seen.add(rt.id);
238
- return true;
239
- });
240
- }
241
-
242
- function generateCrawlTasks(platform, brand, period, outputRoot) {
243
- const tasks = [];
244
- const brandGroups = {};
245
- accountsList.forEach(a => {
246
- if (!brandGroups[a.brand]) brandGroups[a.brand] = { brand: a.brand, brandSlug: a.brand.replace(/ /g, '_'), platforms: [], chromeProfile: a.chrome_profile };
247
- if (!brandGroups[a.brand].platforms.includes(a.platform)) brandGroups[a.brand].platforms.push(a.platform);
248
- });
249
- const targetAccounts = Object.values(brandGroups).filter(a => {
250
- if (brand && a.brand !== brand) return false;
251
- return true;
252
- });
253
-
254
- targetAccounts.forEach(account => {
255
- const targetPlatforms = platform ? [platform] : account.platforms;
256
- targetPlatforms.forEach(p => {
257
- if (!account.platforms.includes(p)) return;
258
- const reports = getReportsForPlatform(p);
259
- reports.forEach(rt => {
260
- const platformDir = p.charAt(0).toUpperCase() + p.slice(1);
261
- const outputPath = path.join(
262
- outputRoot, platformDir, account.brandSlug, rt.id,
263
- `${period}_${account.brandSlug}_${rt.id}.${rt.fileExtension}`
264
- );
265
- tasks.push({
266
- platform: p,
267
- brand: account.brand,
268
- brandSlug: account.brandSlug,
269
- chromeProfile: account.chromeProfile,
270
- reportType: rt,
271
- period,
272
- outputPath
273
- });
274
- });
275
- });
276
- });
277
-
278
- return tasks;
279
- }
280
-
281
- module.exports = { generateCrawlTasks, getReportsForPlatform, getPreviousMonth };
@@ -1,72 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # Session Health Check — Validates if a marketplace session is still active
4
- # =============================================================================
5
- # Returns exit code 0 if session is valid, 1 if expired/invalid.
6
- # Designed to be called by other scripts or directly by AI agents.
7
- #
8
- # Usage:
9
- # ./session-check.sh <platform> <brand-id>
10
- # # Exit 0 = valid, Exit 1 = expired/no-profile
11
- # =============================================================================
12
-
13
- set -uo pipefail
14
-
15
- PROFILE_ROOT="${MARKETPLACE_PROFILE_ROOT:-$HOME/.marketplace-crawler/profiles}"
16
- AB_CMD="${AGENT_BROWSER_CMD:-npx -y agent-browser}"
17
-
18
- platform="${1:-}"
19
- brand_id="${2:-}"
20
-
21
- if [ -z "$platform" ] || [ -z "$brand_id" ]; then
22
- echo "Usage: $0 <platform> <brand-id>"
23
- echo "Example: $0 shopee vita-dairy"
24
- exit 2
25
- fi
26
-
27
- PROFILE_DIR="$PROFILE_ROOT/${platform}-${brand_id}"
28
-
29
- # Check if profile exists
30
- if [ ! -d "$PROFILE_DIR" ]; then
31
- echo "NO_PROFILE"
32
- exit 1
33
- fi
34
-
35
- # Define test URLs and login page indicators
36
- case "$platform" in
37
- shopee)
38
- CHECK_URL="https://banhang.shopee.vn/portal/dashboard/"
39
- LOGIN_PATTERN="buyer/login\|account/login"
40
- ;;
41
- lazada)
42
- CHECK_URL="https://sellercenter.lazada.vn/apps/dashboard"
43
- LOGIN_PATTERN="seller/login\|sso/login"
44
- ;;
45
- tiktok)
46
- CHECK_URL="https://seller-vn.tiktok.com/homepage?shop_region=VN"
47
- LOGIN_PATTERN="account/login\|passport"
48
- ;;
49
- *)
50
- echo "UNKNOWN_PLATFORM"
51
- exit 2
52
- ;;
53
- esac
54
-
55
- # Try to navigate to dashboard and check if redirected to login
56
- $AB_CMD --profile "$PROFILE_DIR" open "$CHECK_URL" 2>/dev/null
57
- sleep 3
58
- $AB_CMD --profile "$PROFILE_DIR" wait --load networkidle 2>/dev/null || true
59
-
60
- CURRENT_URL=$($AB_CMD --profile "$PROFILE_DIR" get url 2>/dev/null || echo "ERROR")
61
- $AB_CMD --profile "$PROFILE_DIR" close 2>/dev/null || true
62
-
63
- if echo "$CURRENT_URL" | grep -qi "$LOGIN_PATTERN"; then
64
- echo "EXPIRED"
65
- exit 1
66
- elif echo "$CURRENT_URL" | grep -qi "error\|ERROR"; then
67
- echo "ERROR"
68
- exit 1
69
- else
70
- echo "VALID"
71
- exit 0
72
- fi
@@ -1,349 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # Session Manager for Marketplace Report Crawler
4
- # =============================================================================
5
- # Cross-agent compatible: works from ANY AI coding agent that can run shell.
6
- # Uses agent-browser CLI for persistent browser sessions.
7
- #
8
- # Usage:
9
- # ./session-manager.sh status # Show all sessions
10
- # ./session-manager.sh login <platform> <brand-id> # Guided login (headed)
11
- # ./session-manager.sh login-all # Login all active accounts
12
- # ./session-manager.sh check <platform> <brand-id> # Check session health
13
- # ./session-manager.sh check-all # Check all sessions
14
- # =============================================================================
15
-
16
- set -euo pipefail
17
-
18
- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
19
- CONFIG_DIR="$SCRIPT_DIR/../config"
20
- ACCOUNTS_FILE="$CONFIG_DIR/accounts.json"
21
- SESSIONS_FILE="$CONFIG_DIR/sessions.json"
22
-
23
- # Default profile root — can be overridden via env
24
- PROFILE_ROOT="${MARKETPLACE_PROFILE_ROOT:-$HOME/.marketplace-crawler/profiles}"
25
-
26
- # agent-browser command — use npx if not installed globally
27
- AB_CMD="${AGENT_BROWSER_CMD:-npx -y agent-browser}"
28
-
29
- # Platform URLs via functions (compatible with bash 3.x on macOS)
30
- get_login_url() {
31
- case "$1" in
32
- shopee) echo "https://banhang.shopee.vn/" ;;
33
- lazada) echo "https://sellercenter.lazada.vn/" ;;
34
- tiktok) echo "https://seller-vn.tiktok.com/" ;;
35
- *) echo "" ;;
36
- esac
37
- }
38
-
39
- get_check_url() {
40
- case "$1" in
41
- shopee) echo "https://banhang.shopee.vn/portal/dashboard/" ;;
42
- lazada) echo "https://sellercenter.lazada.vn/apps/dashboard" ;;
43
- tiktok) echo "https://seller-vn.tiktok.com/homepage?shop_region=VN" ;;
44
- *) echo "" ;;
45
- esac
46
- }
47
-
48
- get_login_indicator() {
49
- case "$1" in
50
- shopee) echo "buyer/login" ;;
51
- lazada) echo "seller/login" ;;
52
- tiktok) echo "account/login" ;;
53
- *) echo "login" ;;
54
- esac
55
- }
56
-
57
- # --- Helpers ---
58
-
59
- get_profile_dir() {
60
- local platform="$1"
61
- local brand_id="$2"
62
- echo "$PROFILE_ROOT/${platform}-${brand_id}"
63
- }
64
-
65
- ensure_profile_dir() {
66
- local dir="$1"
67
- mkdir -p "$dir"
68
- }
69
-
70
- get_brands() {
71
- # Returns brand IDs from accounts.json
72
- node -e "
73
- const a = require('$ACCOUNTS_FILE');
74
- a.accounts.filter(x => x.active).forEach(x => console.log(x.id));
75
- "
76
- }
77
-
78
- get_brand_platforms() {
79
- local brand_id="$1"
80
- node -e "
81
- const a = require('$ACCOUNTS_FILE');
82
- const b = a.accounts.find(x => x.id === '$brand_id');
83
- if (b) b.platforms.forEach(p => console.log(p));
84
- "
85
- }
86
-
87
- get_brand_name() {
88
- local brand_id="$1"
89
- node -e "
90
- const a = require('$ACCOUNTS_FILE');
91
- const b = a.accounts.find(x => x.id === '$brand_id');
92
- if (b) console.log(b.brand);
93
- "
94
- }
95
-
96
- update_session_status() {
97
- local platform="$1"
98
- local brand_id="$2"
99
- local status="$3"
100
- local now
101
- now=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
102
-
103
- # Create sessions.json if not exists
104
- if [ ! -f "$SESSIONS_FILE" ]; then
105
- echo '{"sessions":[]}' > "$SESSIONS_FILE"
106
- fi
107
-
108
- node -e "
109
- const fs = require('fs');
110
- const data = JSON.parse(fs.readFileSync('$SESSIONS_FILE', 'utf8'));
111
- const idx = data.sessions.findIndex(s => s.platform === '$platform' && s.brandId === '$brand_id');
112
- const entry = {
113
- platform: '$platform',
114
- brandId: '$brand_id',
115
- status: '$status',
116
- profilePath: '$(get_profile_dir "$platform" "$brand_id")',
117
- $([ "$status" = "valid" ] && echo "lastLogin: '$now'," || echo "")
118
- lastCheck: '$now'
119
- };
120
- if (idx >= 0) { Object.assign(data.sessions[idx], entry); }
121
- else { data.sessions.push(entry); }
122
- fs.writeFileSync('$SESSIONS_FILE', JSON.stringify(data, null, 2));
123
- "
124
- }
125
-
126
- # --- Commands ---
127
-
128
- cmd_status() {
129
- echo ""
130
- echo "📊 Marketplace Session Status"
131
- echo "═══════════════════════════════════════════════════════"
132
- echo ""
133
- echo "Profile root: $PROFILE_ROOT"
134
- echo ""
135
-
136
- if [ ! -f "$SESSIONS_FILE" ]; then
137
- echo "⚠️ No sessions tracked yet. Run 'login-all' first."
138
- return
139
- fi
140
-
141
- node -e "
142
- const fs = require('fs');
143
- const data = JSON.parse(fs.readFileSync('$SESSIONS_FILE', 'utf8'));
144
- if (data.sessions.length === 0) {
145
- console.log('⚠️ No sessions tracked yet.');
146
- process.exit(0);
147
- }
148
- const icons = { valid: '✅', expired: '❌', unknown: '❓' };
149
- data.sessions.forEach(s => {
150
- const icon = icons[s.status] || '❓';
151
- const last = s.lastCheck ? new Date(s.lastCheck).toLocaleString() : 'never';
152
- console.log(\` \${icon} \${s.platform.padEnd(8)} \${s.brandId.padEnd(15)} \${s.status.padEnd(10)} last: \${last}\`);
153
- });
154
- "
155
- echo ""
156
- }
157
-
158
- cmd_login() {
159
- local platform="$1"
160
- local brand_id="$2"
161
- local brand_name
162
- brand_name=$(get_brand_name "$brand_id")
163
- local profile_dir
164
- profile_dir=$(get_profile_dir "$platform" "$brand_id")
165
- local login_url
166
- login_url=$(get_login_url "$platform")
167
-
168
- ensure_profile_dir "$profile_dir"
169
-
170
- echo ""
171
- echo "🔐 Guided Login: $brand_name on ${platform^^}"
172
- echo "═══════════════════════════════════════════════════════"
173
- echo ""
174
- echo "📂 Profile: $profile_dir"
175
- echo "🌐 URL: $login_url"
176
- echo ""
177
- echo "⏳ Opening browser in headed mode..."
178
- echo " → Please login manually (solve CAPTCHA/OTP as needed)"
179
- echo " → When you see the dashboard, press Enter here to save session"
180
- echo ""
181
-
182
- # Open browser in headed mode with persistent profile
183
- $AB_CMD --profile "$profile_dir" --headed open "$login_url" &
184
- local ab_pid=$!
185
-
186
- # Wait for user to confirm login
187
- read -r -p "✅ Press Enter when login is complete... "
188
-
189
- # Take a screenshot as proof
190
- local screenshot_dir="$profile_dir/screenshots"
191
- mkdir -p "$screenshot_dir"
192
- $AB_CMD --profile "$profile_dir" screenshot "$screenshot_dir/login-success-$(date +%Y%m%d).png" 2>/dev/null || true
193
-
194
- # Close the browser (profile is auto-saved)
195
- $AB_CMD --profile "$profile_dir" close 2>/dev/null || true
196
-
197
- # Update session tracking
198
- update_session_status "$platform" "$brand_id" "valid"
199
-
200
- echo ""
201
- echo "✅ Session saved for $brand_name on ${platform^^}"
202
- echo " Profile: $profile_dir"
203
- echo ""
204
- }
205
-
206
- cmd_login_all() {
207
- echo ""
208
- echo "🔐 Guided Login — All Active Accounts"
209
- echo "═══════════════════════════════════════════════════════"
210
- echo ""
211
-
212
- local brands
213
- brands=$(get_brands)
214
-
215
- for brand_id in $brands; do
216
- local platforms
217
- platforms=$(get_brand_platforms "$brand_id")
218
- for platform in $platforms; do
219
- cmd_login "$platform" "$brand_id"
220
- done
221
- done
222
-
223
- echo ""
224
- echo "✅ All accounts logged in. Run 'check-all' to verify."
225
- echo ""
226
- }
227
-
228
- cmd_check() {
229
- local platform="$1"
230
- local brand_id="$2"
231
- local brand_name
232
- brand_name=$(get_brand_name "$brand_id")
233
- local profile_dir
234
- profile_dir=$(get_profile_dir "$platform" "$brand_id")
235
- local check_url
236
- check_url=$(get_check_url "$platform")
237
- local login_indicator
238
- login_indicator=$(get_login_indicator "$platform")
239
-
240
- if [ ! -d "$profile_dir" ]; then
241
- echo "❌ ${platform}/${brand_id}: No profile found (never logged in)"
242
- update_session_status "$platform" "$brand_id" "unknown"
243
- return 1
244
- fi
245
-
246
- # Open in headless mode, navigate to dashboard, check URL
247
- $AB_CMD --profile "$profile_dir" open "$check_url" 2>/dev/null || true
248
- sleep 3
249
- $AB_CMD --profile "$profile_dir" wait --load networkidle 2>/dev/null || true
250
-
251
- local current_url
252
- current_url=$($AB_CMD --profile "$profile_dir" get url 2>/dev/null || echo "error")
253
-
254
- $AB_CMD --profile "$profile_dir" close 2>/dev/null || true
255
-
256
- if echo "$current_url" | grep -qi "$login_indicator"; then
257
- echo "❌ ${platform}/${brand_id} ($brand_name): Session EXPIRED"
258
- update_session_status "$platform" "$brand_id" "expired"
259
- return 1
260
- else
261
- echo "✅ ${platform}/${brand_id} ($brand_name): Session VALID"
262
- update_session_status "$platform" "$brand_id" "valid"
263
- return 0
264
- fi
265
- }
266
-
267
- cmd_check_all() {
268
- echo ""
269
- echo "🔍 Checking All Sessions"
270
- echo "═══════════════════════════════════════════════════════"
271
- echo ""
272
-
273
- local valid=0
274
- local expired=0
275
- local unknown=0
276
-
277
- local brands
278
- brands=$(get_brands)
279
-
280
- for brand_id in $brands; do
281
- local platforms
282
- platforms=$(get_brand_platforms "$brand_id")
283
- for platform in $platforms; do
284
- if cmd_check "$platform" "$brand_id"; then
285
- ((valid++))
286
- else
287
- ((expired++))
288
- fi
289
- done
290
- done
291
-
292
- echo ""
293
- echo "📊 Summary: $valid valid, $expired expired/unknown"
294
- echo ""
295
- }
296
-
297
- # --- Main ---
298
-
299
- case "${1:-help}" in
300
- status)
301
- cmd_status
302
- ;;
303
- login)
304
- if [ $# -lt 3 ]; then
305
- echo "Usage: $0 login <platform> <brand-id>"
306
- echo "Platforms: shopee, lazada, tiktok"
307
- exit 1
308
- fi
309
- cmd_login "$2" "$3"
310
- ;;
311
- login-all)
312
- cmd_login_all
313
- ;;
314
- check)
315
- if [ $# -lt 3 ]; then
316
- echo "Usage: $0 check <platform> <brand-id>"
317
- exit 1
318
- fi
319
- cmd_check "$2" "$3"
320
- ;;
321
- check-all)
322
- cmd_check_all
323
- ;;
324
- *)
325
- echo ""
326
- echo "🕷️ Marketplace Session Manager"
327
- echo "════════════════════════════════"
328
- echo ""
329
- echo "Commands:"
330
- echo " status Show all session statuses"
331
- echo " login <platform> <brand-id> Guided login (opens browser)"
332
- echo " login-all Login all active accounts"
333
- echo " check <platform> <brand-id> Check if session is valid"
334
- echo " check-all Check all sessions"
335
- echo ""
336
- echo "Platforms: shopee, lazada, tiktok"
337
- echo "Brands: $(get_brands | tr '\n' ', ')"
338
- echo ""
339
- echo "Environment:"
340
- echo " MARKETPLACE_PROFILE_ROOT Profile directory (default: ~/.marketplace-crawler/profiles)"
341
- echo " AGENT_BROWSER_CMD agent-browser command (default: npx -y agent-browser)"
342
- echo ""
343
- echo "Examples:"
344
- echo " $0 login shopee vita-dairy"
345
- echo " $0 check-all"
346
- echo " $0 status"
347
- echo ""
348
- ;;
349
- esac