adaptive-memory-multi-model-router 2.14.7 → 2.14.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.publish-tick +1 -1
- package/README.md +13 -17
- package/assets/demo-hn.gif +0 -0
- package/bin/a3m-upgrade-check +18 -0
- package/demo/asciinema-demo.sh +47 -36
- package/demo/demo-hn.tape +57 -78
- package/demo/recording.cast +55 -0
- package/dist/benchmark/reproducible.d.ts +55 -0
- package/dist/benchmark/reproducible.js +172 -0
- package/dist/benchmark/reproducible.js.map +1 -0
- package/dist/cli.js +173 -34
- package/dist/observability/changeWatch.d.ts +15 -0
- package/dist/observability/changeWatch.js +92 -0
- package/dist/observability/changeWatch.js.map +1 -0
- package/dist/observability/fatigueDetector.d.ts +14 -0
- package/dist/observability/fatigueDetector.js +48 -0
- package/dist/observability/fatigueDetector.js.map +1 -0
- package/dist/routing/advancedRouter.js +11 -1
- package/dist/routing/crossModelValidation.d.ts +10 -0
- package/dist/routing/crossModelValidation.js +44 -0
- package/dist/routing/crossModelValidation.js.map +1 -0
- package/dist/scripts/banner.js +29 -0
- package/docs/ANALYSIS_PRINCIPLES.md +162 -0
- package/docs/FACTORY_RESET.md +34 -0
- package/docs/GEO_OPTIMIZATION.md +30 -0
- package/docs/MIDDLEWARE_CHAIN.md +35 -0
- package/docs/PROMO_CHECKLIST.md +200 -0
- package/docs/ROUTING_RUBRIC.md +197 -0
- package/docs/SOCIAL_LISTENING.md +219 -0
- package/eval/evals.json +199 -0
- package/package.json +1 -1
- package/research/PUBLISH_LOG.md +2 -2
- package/scripts/content-planner.js +25 -0
- package/src/benchmark/reproducible.ts +246 -0
- package/src/observability/changeWatch.ts +62 -0
- package/src/observability/fatigueDetector.ts +58 -0
- package/src/routing/crossModelValidation.ts +53 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.checkProviderFatigue = checkProviderFatigue;
|
|
4
|
+
exports.formatFatigueReport = formatFatigueReport;
|
|
5
|
+
const metrics_1 = require("./metrics");
|
|
6
|
+
function checkProviderFatigue() {
|
|
7
|
+
const allMetrics = (0, metrics_1.getMetrics)().getMetrics();
|
|
8
|
+
const byProvider = {};
|
|
9
|
+
for (const m of allMetrics) {
|
|
10
|
+
const p = (m.labels || {}).provider;
|
|
11
|
+
if (!p)
|
|
12
|
+
continue;
|
|
13
|
+
if (!byProvider[p])
|
|
14
|
+
byProvider[p] = { req: 0, err: 0, lat: [] };
|
|
15
|
+
if (m.type === 'histogram' && m.name.includes('latency') && typeof m.value === 'number')
|
|
16
|
+
byProvider[p].lat.push(m.value * 1000);
|
|
17
|
+
}
|
|
18
|
+
const reports = [];
|
|
19
|
+
let anyAction = false;
|
|
20
|
+
for (const [provider, data] of Object.entries(byProvider)) {
|
|
21
|
+
const errRate = data.req > 0 ? data.err / data.req : 0;
|
|
22
|
+
const healthy = errRate < 0.1;
|
|
23
|
+
if (!healthy)
|
|
24
|
+
anyAction = true;
|
|
25
|
+
reports.push({
|
|
26
|
+
provider, queriesCount: data.req, errorRate: errRate,
|
|
27
|
+
healthy,
|
|
28
|
+
recommendedAction: healthy ? 'No action needed' : `Error rate ${(errRate * 100).toFixed(1)}% — add fallback`,
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
reports,
|
|
33
|
+
summary: anyAction ? '⚠️ Provider fatigue detected' : '✅ All providers healthy',
|
|
34
|
+
anyActionNeeded: anyAction,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function formatFatigueReport() {
|
|
38
|
+
const r = checkProviderFatigue();
|
|
39
|
+
let out = ` ${r.summary}\n Checked ${r.reports.length} providers\n\n`;
|
|
40
|
+
for (const rep of r.reports) {
|
|
41
|
+
out += ` ${rep.healthy ? '✅' : '⚠️'} ${rep.provider}\n Queries: ${rep.queriesCount} | Errors: ${(rep.errorRate * 100).toFixed(1)}%\n`;
|
|
42
|
+
if (!rep.healthy)
|
|
43
|
+
out += ` ⚠️ ${rep.recommendedAction}\n`;
|
|
44
|
+
out += '\n';
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=fatigueDetector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fatigueDetector.js","sourceRoot":"","sources":["../../src/observability/fatigueDetector.ts"],"names":[],"mappings":";;AAgBA,oDA8BC;AAED,kDASC;AAzDD,uCAAuC;AAgBvC,SAAgB,oBAAoB;IAClC,MAAM,UAAU,GAAG,IAAA,oBAAU,GAAE,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAgE,EAAE,CAAC;IAEnF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC;QACpC,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAChE,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;YAAE,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IAClI,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,OAAO,GAAG,GAAG,CAAC;QAC9B,IAAI,CAAC,OAAO;YAAE,SAAS,GAAG,IAAI,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC;YACX,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO;YACpD,OAAO;YACP,iBAAiB,EAAE,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,GAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB;SAC3G,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,OAAO;QACP,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,yBAAyB;QAC/E,eAAe,EAAE,SAAS;KAC3B,CAAC;AACJ,CAAC;AAED,SAAgB,mBAAmB;IACjC,MAAM,CAAC,GAAG,oBAAoB,EAAE,CAAC;IACjC,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,OAAO,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,gBAAgB,CAAC;IACxE,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QAC5B,GAAG,IAAI,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,QAAQ,mBAAmB,GAAG,CAAC,YAAY,cAAc,CAAC,GAAG,CAAC,SAAS,GAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;QACzI,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,GAAG,IAAI,YAAY,GAAG,CAAC,iBAAiB,IAAI,CAAC;QAC/D,GAAG,IAAI,IAAI,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -79,7 +79,13 @@ function buildModelProfiles() {
|
|
|
79
79
|
return profiles;
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
var MODEL_PROFILES = {};
|
|
83
|
+
try {
|
|
84
|
+
MODEL_PROFILES = buildModelProfiles();
|
|
85
|
+
} catch (e) {
|
|
86
|
+
// Circular dependency at module load — will retry on first use
|
|
87
|
+
MODEL_PROFILES = {};
|
|
88
|
+
}
|
|
83
89
|
|
|
84
90
|
// Refresh profiles when providers change
|
|
85
91
|
function refreshModelProfiles() {
|
|
@@ -88,6 +94,10 @@ function refreshModelProfiles() {
|
|
|
88
94
|
|
|
89
95
|
exports.MODEL_PROFILES = MODEL_PROFILES;
|
|
90
96
|
|
|
97
|
+
// Ensure exports stay in sync if profiles are rebuilt
|
|
98
|
+
function updateExports() {
|
|
99
|
+
exports.MODEL_PROFILES = MODEL_PROFILES;
|
|
100
|
+
}
|
|
91
101
|
// ============================================================
|
|
92
102
|
// FEATURE EXTRACTION (v3 — multi-signal complexity scorer)
|
|
93
103
|
// ============================================================
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface ValidationResult {
|
|
2
|
+
approved: boolean;
|
|
3
|
+
selectedProvider: string;
|
|
4
|
+
validatedProvider: string;
|
|
5
|
+
reason: string;
|
|
6
|
+
costOverhead: number;
|
|
7
|
+
}
|
|
8
|
+
export declare function validateRouting(query: string, selectedProvider: string, selectedModel: string, options?: {
|
|
9
|
+
validatorProvider?: string;
|
|
10
|
+
}): Promise<ValidationResult>;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.validateRouting = validateRouting;
|
|
4
|
+
const providerConfig_1 = require("../providers/providerConfig");
|
|
5
|
+
const metrics_1 = require("../observability/metrics");
|
|
6
|
+
async function validateRouting(query, selectedProvider, selectedModel, options) {
|
|
7
|
+
const metrics = (0, metrics_1.getMetrics)();
|
|
8
|
+
const providers = (0, providerConfig_1.getAvailableProviders)();
|
|
9
|
+
const validatorId = options?.validatorProvider || pickValidator(selectedProvider, providers);
|
|
10
|
+
const validationPrompt = `A developer asked: "${query.slice(0, 200)}"
|
|
11
|
+
The AI router selected: ${selectedProvider}/${selectedModel}
|
|
12
|
+
Was this the RIGHT choice? Answer YES or NO first, then explain in ONE sentence.`;
|
|
13
|
+
try {
|
|
14
|
+
const validatorProvider = providers[validatorId];
|
|
15
|
+
if (!validatorProvider) {
|
|
16
|
+
metrics.incrementCounter('a3m_validation_skipped', { reason: 'no_validator' });
|
|
17
|
+
return { approved: true, selectedProvider, validatedProvider: 'none', reason: 'No validator available', costOverhead: 0 };
|
|
18
|
+
}
|
|
19
|
+
const startTime = Date.now();
|
|
20
|
+
const response = await validatorProvider.callProvider(selectedModel, validationPrompt, 50);
|
|
21
|
+
const elapsed = (Date.now() - startTime) / 1000;
|
|
22
|
+
const text = String(response?.content || response?.text || '').trim();
|
|
23
|
+
const approved = text.startsWith('YES') || text.startsWith('yes');
|
|
24
|
+
const reason = text.replace(/^(YES|NO)\s*\|?\s*/, '').trim() || text.slice(0, 100);
|
|
25
|
+
metrics.incrementCounter('a3m_validation_total');
|
|
26
|
+
if (approved)
|
|
27
|
+
metrics.incrementCounter('a3m_validation_approved');
|
|
28
|
+
else
|
|
29
|
+
metrics.incrementCounter('a3m_validation_rejected');
|
|
30
|
+
metrics.recordHistogram('a3m_validation_latency_seconds', elapsed);
|
|
31
|
+
return { approved, selectedProvider, validatedProvider: validatorId, reason, costOverhead: 0.0001 };
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
metrics.incrementCounter('a3m_validation_error');
|
|
35
|
+
return { approved: true, selectedProvider, validatedProvider: 'error', reason: 'Validation failed', costOverhead: 0 };
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function pickValidator(selectedProvider, providers) {
|
|
39
|
+
const ids = Object.keys(providers).filter(id => id !== selectedProvider);
|
|
40
|
+
if (ids.length === 0)
|
|
41
|
+
return selectedProvider;
|
|
42
|
+
return ids[0];
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=crossModelValidation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crossModelValidation.js","sourceRoot":"","sources":["../../src/routing/crossModelValidation.ts"],"names":[],"mappings":";;AAWA,0CAmCC;AA9CD,gEAAoE;AACpE,sDAAsD;AAU/C,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,gBAAwB,EACxB,aAAqB,EACrB,OAAwC;IAExC,MAAM,OAAO,GAAG,IAAA,oBAAU,GAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAA,sCAAqB,GAAE,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,EAAE,iBAAiB,IAAI,aAAa,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;IAE7F,MAAM,gBAAgB,GAAG,uBAAuB,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;0BAC3C,gBAAgB,IAAI,aAAa;iFACsB,CAAC;IAEhF,IAAI,CAAC;QACH,MAAM,iBAAiB,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;QACjD,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,OAAO,CAAC,gBAAgB,CAAC,wBAAwB,EAAE,EAAE,MAAM,EAAE,cAAqB,EAAE,CAAC,CAAC;YACtF,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QAC5H,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAO,iBAAyB,CAAC,YAAY,CAAC,aAAa,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAC;QACpG,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,OAAO,IAAI,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACnF,OAAO,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,CAAC;QACjD,IAAI,QAAQ;YAAE,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC;;YAC7D,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC;QACzD,OAAO,CAAC,eAAe,CAAC,gCAAgC,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;IACtG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,CAAC;QACjD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;IACxH,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,gBAAwB,EAAE,SAA8B;IAC7E,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,gBAAgB,CAAC,CAAC;IACzE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,gBAAgB,CAAC;IAC9C,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* A3M Router — Terminal ASCII Art Banner
|
|
4
|
+
*
|
|
5
|
+
* Printed on CLI startup to reinforce A3M branding.
|
|
6
|
+
* Usage:
|
|
7
|
+
* node scripts/banner.js
|
|
8
|
+
* // or import './banner' in CLI entry point
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const A3M_BANNER = `
|
|
12
|
+
╔══════════════════════════════════════════════════════════╗
|
|
13
|
+
║ ╔═╗╔═╗╔╗╔╔═╗ ║
|
|
14
|
+
║ ╠═╣║ ║║║║║ ║ ║
|
|
15
|
+
║ ╩ ╩╚═╝╝╚╝╚═╝ ║
|
|
16
|
+
║ ║
|
|
17
|
+
║ Parallel Multi-LLM Execution Engine ║
|
|
18
|
+
║ ║
|
|
19
|
+
║ 47+ Providers · Ensemble Voting · 62% Cost Savings ║
|
|
20
|
+
║ ║
|
|
21
|
+
║ ${'\x1b[2m'}https://github.com/Das-rebel/a3m-router${'\x1b[0m'}${' '.repeat(19)}║
|
|
22
|
+
╚══════════════════════════════════════════════════════════╝
|
|
23
|
+
`;
|
|
24
|
+
|
|
25
|
+
module.exports = A3M_BANNER;
|
|
26
|
+
|
|
27
|
+
if (require.main === module) {
|
|
28
|
+
process.stdout.write(A3M_BANNER);
|
|
29
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# A3M Router — Analysis Principles
|
|
2
|
+
|
|
3
|
+
These principles apply to every routing decision, performance analysis, and optimization recommendation. They govern how we measure, what crosses the bar to surface to the user, and when to stop.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Evidence Is the Bar
|
|
8
|
+
|
|
9
|
+
Every claim about routing performance must cite specific data from the actual router:
|
|
10
|
+
|
|
11
|
+
- Name the query type (trivial, simple, moderate, complex, expert)
|
|
12
|
+
- Cite the cost, latency, and provider for each routing decision
|
|
13
|
+
- If you don't have the data to support a claim, pull it before making the claim
|
|
14
|
+
- "Industry typically shows X" or "theory suggests" is not evidence
|
|
15
|
+
- When recommending a routing change, separately show the data that would falsify the recommendation if it existed
|
|
16
|
+
- "Looks slow" / "seems expensive" / "could be improved" is a draft, not a finding
|
|
17
|
+
|
|
18
|
+
When data is too thin to support a recommendation, say so explicitly. Don't paper over uncertainty.
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## 2. STOP Conditions
|
|
23
|
+
|
|
24
|
+
The following conditions halt routing and surface a blocking error BEFORE any provider is called:
|
|
25
|
+
|
|
26
|
+
| Condition | Action |
|
|
27
|
+
|-----------|--------|
|
|
28
|
+
| **No providers configured** | STOP. Show setup wizard. Recommend `a3m-router setup` |
|
|
29
|
+
| **All API keys expired or missing** | STOP. List which keys are needed. Show env var names. |
|
|
30
|
+
| **Zero remaining budget** (budget cap hit) | STOP. Show spending summary. Offer to increase cap. |
|
|
31
|
+
| **Provider health check all red** | STOP. Show health report. Offer to retry after 60s. |
|
|
32
|
+
| **Circuit breaker open on all providers** | STOP. Show which providers are down. Show estimated recovery time. |
|
|
33
|
+
| **Rate limit exceeded on all available providers** | STOP. Show backoff time. Offer to queue query. |
|
|
34
|
+
| **Query contains flagged content** (PII, injection attempt, etc.) | STOP. Show guardrails violated. Do NOT route. |
|
|
35
|
+
|
|
36
|
+
Do NOT silently degrade — stop, explain why, and offer a path forward.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## 3. Statistical Significance Gate
|
|
41
|
+
|
|
42
|
+
Before reporting a routing accuracy improvement or degradation:
|
|
43
|
+
|
|
44
|
+
- **Minimum 100 queries** for any accuracy claim. Fewer than 100 ±1 tier hits is too noisy. Say "insufficient data" instead of reporting a number.
|
|
45
|
+
- **Minimum 50 queries per query type** for per-type accuracy breakdown. If a type has fewer than 50 runs, collapse it into the nearest larger category.
|
|
46
|
+
- **Minimum 7 days or 500 queries** before claiming a cost savings improvement. Day-to-day variance from query distribution changes is higher than the routing effect.
|
|
47
|
+
- **Minimum 14 days or 1000 queries** before comparing two routing configurations (e.g., keyword-only vs ML-assisted).
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## 4. Never Route to an Untested Provider Without Fallback
|
|
52
|
+
|
|
53
|
+
When adding a new provider to the routing pool:
|
|
54
|
+
|
|
55
|
+
1. First test the provider via `a3m-router test <provider>` — must pass health check
|
|
56
|
+
2. Route only queries with complexity < 30 to the new provider for first 50 queries (proving phase)
|
|
57
|
+
3. After 50 queries with <10% error rate, promote to full routing pool
|
|
58
|
+
4. Always pair a new provider with a mature fallback
|
|
59
|
+
|
|
60
|
+
If a provider has no proven track record in this A3M installation, it must have:
|
|
61
|
+
- A verified API key (checked at startup)
|
|
62
|
+
- A health check pass within the last 15 minutes
|
|
63
|
+
- An active circuit breaker with <3 trips in the last hour
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## 5. Confirmation Before Bulk Operations
|
|
68
|
+
|
|
69
|
+
Before routing more than 10 concurrent queries through a new configuration:
|
|
70
|
+
|
|
71
|
+
- Show the count, breakdown by query type, and expected cost
|
|
72
|
+
- Show the pre/post cost comparison if the change would affect routing
|
|
73
|
+
- Ask for confirmation before proceeding
|
|
74
|
+
|
|
75
|
+
Exception: Automated cache warming and health check pings do not require confirmation.
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## 6. Change Tracking Requirement
|
|
80
|
+
|
|
81
|
+
Every routing decision must be logged with:
|
|
82
|
+
|
|
83
|
+
- Timestamp
|
|
84
|
+
- Query (hashed/no PII for privacy)
|
|
85
|
+
- Query type classification
|
|
86
|
+
- Selected provider
|
|
87
|
+
- Provider tier
|
|
88
|
+
- Actual cost
|
|
89
|
+
- Latency
|
|
90
|
+
- Cache hit/miss
|
|
91
|
+
- Error (if any)
|
|
92
|
+
- Fallback provider used (if any)
|
|
93
|
+
|
|
94
|
+
The audit log is stored in `~/.a3m-router/audit-log.ndjson` — one JSON object per line.
|
|
95
|
+
|
|
96
|
+
Never route without logging. If the audit log file cannot be written, log to stderr and surface a warning.
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
## 7. Signal-Failure Override
|
|
101
|
+
|
|
102
|
+
When a signal that normally contributes to routing decisions is unavailable:
|
|
103
|
+
|
|
104
|
+
| Missing Signal | Override Behavior |
|
|
105
|
+
|---------------|-------------------|
|
|
106
|
+
| **Provider health data** | Assume healthy. Do not penalize the provider. Issue warning. |
|
|
107
|
+
| **Cost data for a provider** | Use the provider's default cost tier. Issue warning. |
|
|
108
|
+
| **Historical accuracy data** | Use the model's global default accuracy. Issue warning. |
|
|
109
|
+
| **Cache** | Route as if cache miss. No penalty to the scoring. |
|
|
110
|
+
| **Budget enforcement data** | Use last-known budget snapshot. If none available, do not enforce budget. Issue warning. |
|
|
111
|
+
|
|
112
|
+
Do NOT fabricate data. Report the override explicitly: "Provider health unavailable — assumed healthy."
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## 8. Read Correlates, Write Commits
|
|
117
|
+
|
|
118
|
+
- **Routing decisions are reads** — they select a provider but don't change the router's behavior
|
|
119
|
+
- **Configuration changes are writes** — adding/removing providers, changing weights, updating thresholds
|
|
120
|
+
- Every configuration change must be logged with old and new values
|
|
121
|
+
- Every configuration change must be reversible within 5 minutes
|
|
122
|
+
- Configuration changes should be tested with at least 10 queries before switching to production
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## 9. Data Freshness Rules
|
|
127
|
+
|
|
128
|
+
| Data Type | Max Age Before Refresh | Behavior When Stale |
|
|
129
|
+
|-----------|----------------------|---------------------|
|
|
130
|
+
| Provider health | 60 seconds | Mark as untested (see rule 4) |
|
|
131
|
+
| Cost data | 24 hours | Use last-known, issue warning |
|
|
132
|
+
| RouterArena score | 7 days | Accept cached, prompt refresh |
|
|
133
|
+
| Model capability profiles | 30 days | Prompt refresh, use cached |
|
|
134
|
+
| Cache entries | Per TTL config | Evict, route normally |
|
|
135
|
+
| Budget state | 1 second | Block if over, allow if under |
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## 10. When You're Unsure
|
|
140
|
+
|
|
141
|
+
- Surface uncertainty in the report. "Thin data" is better than a fabricated number.
|
|
142
|
+
- Ask one targeted question if it would change the recommendation materially. Don't ask for context the data already gives you.
|
|
143
|
+
- If routing quality depends on query distribution (which changes over time) and the distribution has shifted, name what changed and offer to re-profile the golden route set.
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## Quick Reference: Decision Flow
|
|
148
|
+
|
|
149
|
+
```
|
|
150
|
+
Query arrives
|
|
151
|
+
→ Guardrails check (Rule 2)
|
|
152
|
+
→ Health check (Rule 4)
|
|
153
|
+
→ Budget check (Rule 2)
|
|
154
|
+
→ Signal-failure override check (Rule 7)
|
|
155
|
+
→ Cache lookup (Rule 1)
|
|
156
|
+
→ Complexity scoring
|
|
157
|
+
→ Provider selection (Rule 3)
|
|
158
|
+
→ Route & log (Rule 6)
|
|
159
|
+
→ Fallback if failure (Rule 4)
|
|
160
|
+
→ Return response or STOP (Rule 2)
|
|
161
|
+
→ Update metrics (Rule 1)
|
|
162
|
+
```
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# A3M Router — Factory Reset & Recalibration
|
|
2
|
+
|
|
3
|
+
## When to Run
|
|
4
|
+
- Added 3+ new providers since initial setup
|
|
5
|
+
- Changed API keys for 2+ providers
|
|
6
|
+
- RouterArena score changed significantly
|
|
7
|
+
- More than 30 days since last setup
|
|
8
|
+
|
|
9
|
+
## How
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
a3m-router setup --fresh
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
This will:
|
|
16
|
+
1. Clear old provider weights and thresholds
|
|
17
|
+
2. Re-scan environment variables for API keys
|
|
18
|
+
3. Re-test all configured providers
|
|
19
|
+
4. Recalibrate routing weights based on:
|
|
20
|
+
- Current provider latency
|
|
21
|
+
- Current provider availability
|
|
22
|
+
- Current pricing
|
|
23
|
+
5. Save new config
|
|
24
|
+
|
|
25
|
+
## What Gets Reset
|
|
26
|
+
|
|
27
|
+
| Config | Reset? | New Value |
|
|
28
|
+
|--------|--------|-----------|
|
|
29
|
+
| Provider weights | Yes | Equal weight for all working providers |
|
|
30
|
+
| Budget caps | No | Kept |
|
|
31
|
+
| Cache | No | Preserved |
|
|
32
|
+
| Health scores | Yes | Re-tested from scratch |
|
|
33
|
+
| Audit log | No | Preserved |
|
|
34
|
+
| Change log | No | Preserved |
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# A3M Router — GEO Optimization for AI Search
|
|
2
|
+
|
|
3
|
+
Uses Generative Engine Optimization (GEO) so AI answer engines cite A3M.
|
|
4
|
+
Based on Princeton/GA Tech GEO (KDD 2024, arXiv:2311.09735).
|
|
5
|
+
|
|
6
|
+
## Signal Stack
|
|
7
|
+
|
|
8
|
+
| Signal | Lift | Applied In |
|
|
9
|
+
|--------|------|-----------|
|
|
10
|
+
| Quotation Addition | +41% | README hero (RouterArena quote) |
|
|
11
|
+
| Statistics Addition | +30% | README ($0.047, 213x, 62%) |
|
|
12
|
+
| Cite Sources | +28% | arXiv link, PR link |
|
|
13
|
+
| Technical Terms | +18% | confidence-weighted voting, semantic routing |
|
|
14
|
+
| Fluency Optimization | +28% | All docs |
|
|
15
|
+
|
|
16
|
+
## PAWC Optimization
|
|
17
|
+
|
|
18
|
+
Position-Adjusted Word Count: sentence #1 worth ~5x sentence #20.
|
|
19
|
+
README first 200 words packed with: arXiv citation, RouterArena score, cost numbers.
|
|
20
|
+
|
|
21
|
+
## Files Optimized
|
|
22
|
+
|
|
23
|
+
- `README.md` — Starts with arXiv citation + scores + stats
|
|
24
|
+
- `docs/comparison-litellm.md` — Precise side-by-side numbers
|
|
25
|
+
- `docs/cost-chart-ascii.md` — AI-extractable data
|
|
26
|
+
- `docs/benchmark.html` — Structured data
|
|
27
|
+
|
|
28
|
+
## Verify
|
|
29
|
+
|
|
30
|
+
Ask any AI: "What's the cheapest and best LLM router?" — A3M should appear.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# A3M Router — Middleware Chain
|
|
2
|
+
|
|
3
|
+
Pre-flight checks before every routing decision.
|
|
4
|
+
|
|
5
|
+
## Chain Order
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
1. Guardrails Check → 9 STOP conditions (docs/ANALYSIS_PRINCIPLES.md)
|
|
9
|
+
2. Provider Health → Quick ping all providers (<50ms)
|
|
10
|
+
3. Budget Check → Remaining budget vs estimated cost (<1ms)
|
|
11
|
+
4. Version Check → npm upgrade available? (cached 24h)
|
|
12
|
+
5. Cache Lookup → Semantic cache for repeated queries (<5ms)
|
|
13
|
+
6. Route → Execute → Call selected provider(s)
|
|
14
|
+
7. Log → Write audit-log.ndjson entry
|
|
15
|
+
8. Cross-Validate → If --validate flag, call second provider (<500ms)
|
|
16
|
+
9. Return Response
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Implementation Status
|
|
20
|
+
|
|
21
|
+
| Step | Status | File |
|
|
22
|
+
|------|--------|------|
|
|
23
|
+
| 1. Guardrails | ✅ | docs/ANALYSIS_PRINCIPLES.md |
|
|
24
|
+
| 2. Health | ✅ | src/routing/providerHealth.ts |
|
|
25
|
+
| 3. Budget | ✅ | src/cost/budgetEnforcer.ts |
|
|
26
|
+
| 4. Version | ✅ | bin/a3m-upgrade-check |
|
|
27
|
+
| 5. Cache | ✅ | src/cache/semanticCache.ts |
|
|
28
|
+
| 6. Route | ✅ | src/routing/advancedRouter.ts |
|
|
29
|
+
| 7. Log | ⚠️ | ~/.a3m-router/audit-log.ndjson |
|
|
30
|
+
| 8. Validate | ✅ | src/routing/crossModelValidation.ts |
|
|
31
|
+
| 9. Return | ✅ | CLI built-in |
|
|
32
|
+
|
|
33
|
+
## Why
|
|
34
|
+
|
|
35
|
+
Every link is a reliability guarantee. Broken chain = charged for failed queries.
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# A3M Router — 40 Free Promotion Channels
|
|
2
|
+
|
|
3
|
+
Ranked by Domain Rating (DR). One action per day = launched in 40 days.
|
|
4
|
+
|
|
5
|
+
## Tier 1: Must-Do (DR 85+) [5 sites]
|
|
6
|
+
|
|
7
|
+
### 1. Product Hunt (DR 91)
|
|
8
|
+
- **Submit:** https://www.producthunt.com/posts/new
|
|
9
|
+
- **Hook:** "Same answer as GPT-5. 200× cheaper. Open source."
|
|
10
|
+
- **Best time:** Tue-Thu, 12:01am PT
|
|
11
|
+
- **Tips:** Post maker comment within 1hr. Reply to every comment. Have 5 friends ready to upvote.
|
|
12
|
+
- **❌** Don't beg for upvotes. Don't launch Friday-Sunday.
|
|
13
|
+
|
|
14
|
+
### 2. Hacker News (DR 91)
|
|
15
|
+
- **Submit:** https://news.ycombinator.com/submit
|
|
16
|
+
- **Hook:** "Show HN: A3M Router – Open-source LLM router, #1 on RouterArena"
|
|
17
|
+
- **Best time:** Tue-Thu, 8:30-10am ET
|
|
18
|
+
- **Tips:** Title must be factual. Be in comments within 30min. Pre-write your first comment explaining the project.
|
|
19
|
+
- **❌** No marketing speak, no $ figures in title, no cherry-picked numbers.
|
|
20
|
+
|
|
21
|
+
### 3. Buy Me a Coffee (DR 91)
|
|
22
|
+
- **Create:** https://buymeacoffee.com
|
|
23
|
+
- **Hook:** "Support open-source AI routing infrastructure"
|
|
24
|
+
- **Tips:** Link from GitHub repo README. Offer "Sponsor" badge for backers.
|
|
25
|
+
|
|
26
|
+
### 4. Crunchbase (DR 90)
|
|
27
|
+
- **Submit:** https://www.crunchbase.com/#/home/index
|
|
28
|
+
- **Hook:** Create company profile for the project
|
|
29
|
+
- **Tips:** Add GitHub link, description, and team. Good for SEO backlinks.
|
|
30
|
+
|
|
31
|
+
### 5. GitHub Trending
|
|
32
|
+
- **Optimize:** Add relevant topics: `llm`, `router`, `ai-gateway`, `open-source`, `model-routing`
|
|
33
|
+
- **Hook:** Good README + active commits = trending chance
|
|
34
|
+
- **Tips:** Star history matters. Consistent commits = algorithmic boost.
|
|
35
|
+
|
|
36
|
+
## Tier 2: High Impact (DR 70-84) [10 sites]
|
|
37
|
+
|
|
38
|
+
### 6. DevTo (DR 85)
|
|
39
|
+
- **Submit:** https://dev.to/new
|
|
40
|
+
- **Hook:** "I built an open-source LLM router that saved me $800/month"
|
|
41
|
+
- **Tags:** `#opensource` `#ai` `#llm` `#nodejs` `#javascript`
|
|
42
|
+
- **❌** No direct product pitches. Share the story, mention the tool.
|
|
43
|
+
|
|
44
|
+
### 7. Reddit — r/programming (DR 84)
|
|
45
|
+
- **Submit:** https://www.reddit.com/r/programming/submit
|
|
46
|
+
- **Hook:** "Show HN-style with technical details"
|
|
47
|
+
- **❌** No self-promo. Frame as "I built this, here's how it works technically."
|
|
48
|
+
|
|
49
|
+
### 8. Reddit — r/MachineLearning (DR 84)
|
|
50
|
+
- **Hook:** Focus on routing algorithms, not the tool
|
|
51
|
+
- **Tips:** Share benchmark methodology. Reference RouterArena.
|
|
52
|
+
|
|
53
|
+
### 9. Reddit — r/commandline (DR 84)
|
|
54
|
+
- **Hook:** "CLI tool to route queries across 47 LLM providers"
|
|
55
|
+
- **Tips:** Show asciinema demo gif.
|
|
56
|
+
|
|
57
|
+
### 10. Reddit — r/node (DR 84)
|
|
58
|
+
- **Hook:** "npm package for intelligent LLM routing — A3M Router"
|
|
59
|
+
- **Tips:** Show the npm install + code example.
|
|
60
|
+
|
|
61
|
+
### 11. Reddit — r/opensource (DR 84)
|
|
62
|
+
- **Hook:** "MIT-licensed open-source LLM router"
|
|
63
|
+
- **Tips:** Emphasize transparency, no vendor lock-in.
|
|
64
|
+
|
|
65
|
+
### 12. LinkedIn (DR 86)
|
|
66
|
+
- **Post:** Write article about the $800/month → $5 story
|
|
67
|
+
- **Tips:** Tag people in the routing space. Use the benchmark chart.
|
|
68
|
+
|
|
69
|
+
### 13. Twitter/X (DR 86)
|
|
70
|
+
- **Thread:** Cost comparison + benchmark + code example
|
|
71
|
+
- **Best time:** 8-10am ET
|
|
72
|
+
- **Tips:** Include a screenshot of the CLI in action.
|
|
73
|
+
|
|
74
|
+
### 14. StackShare (DR 80)
|
|
75
|
+
- **Submit:** https://stackshare.io/add-tool
|
|
76
|
+
- **Hook:** "Intelligent LLM routing for your stack"
|
|
77
|
+
- **Tips:** Compare with LiteLLM, Portkey, OpenRouter.
|
|
78
|
+
|
|
79
|
+
### 15. AlternativeTo (DR 79)
|
|
80
|
+
- **Submit:** https://alternativeto.net/software/new/
|
|
81
|
+
- **Hook:** Alternative to LiteLLM, Portkey, OpenRouter
|
|
82
|
+
- **Tips:** List accurate features and pricing (free/open source).
|
|
83
|
+
|
|
84
|
+
## Tier 3: Niche Communities (DR 50-69) [15 sites]
|
|
85
|
+
|
|
86
|
+
### 16. Lobsters (DR 55)
|
|
87
|
+
- **Submit:** https://lobste.rs
|
|
88
|
+
- **Hook:** Technical discussion about routing algorithms
|
|
89
|
+
- **❌** Tech-heavy crowd. Focus on engineering, not marketing.
|
|
90
|
+
|
|
91
|
+
### 17. Terminal Trove (DR ~65)
|
|
92
|
+
- **Submit:** https://terminaltrove.com
|
|
93
|
+
- **Hook:** CLI tool directory
|
|
94
|
+
|
|
95
|
+
### 18. Awesome LLM Routing (GitHub)
|
|
96
|
+
- **Submit:** PR to add A3M Router
|
|
97
|
+
- **Search:** `awesome-llm` `awesome-ai` `awesome-router`
|
|
98
|
+
|
|
99
|
+
### 19. Show HN on Lobsters
|
|
100
|
+
- **Submit:** https://lobste.rs/stories/new
|
|
101
|
+
- **Hook:** Same as HN but more technical audience
|
|
102
|
+
|
|
103
|
+
### 20. BetaList (DR 58)
|
|
104
|
+
- **Submit:** https://betalist.com/submit
|
|
105
|
+
- **Hook:** "AI-powered LLM router — same answers, 200× cheaper"
|
|
106
|
+
|
|
107
|
+
### 21. BetaPage (DR 52)
|
|
108
|
+
- **Submit:** https://betapage.co/submit-startup
|
|
109
|
+
|
|
110
|
+
### 22. LaunchingNext (DR 60)
|
|
111
|
+
- **Submit:** https://launchingnext.com/submit
|
|
112
|
+
|
|
113
|
+
### 23. SaaSHub (DR 69)
|
|
114
|
+
- **Submit:** https://www.saashub.com/submit
|
|
115
|
+
- **Hook:** Alternative to expensive LLM gateways
|
|
116
|
+
|
|
117
|
+
### 24. FutureTools (DR 62)
|
|
118
|
+
- **Submit:** https://futuretools.io/submit
|
|
119
|
+
|
|
120
|
+
### 25. There's An AI For That (DR 66)
|
|
121
|
+
- **Submit:** https://theresanaiforthat.com/submit/
|
|
122
|
+
|
|
123
|
+
### 26. AI Directory
|
|
124
|
+
- **Search and submit** to top AI directories
|
|
125
|
+
|
|
126
|
+
### 27. npm — Optimize listing
|
|
127
|
+
- **Keywords:** `llm-router`, `ai-gateway`, `model-routing`, `openai-alternative`
|
|
128
|
+
- **Description:** "Intelligent LLM routing across 47 providers. Same answers, 200× cheaper."
|
|
129
|
+
- **Tips:** Good README = higher npm install conversion
|
|
130
|
+
|
|
131
|
+
### 28. GitHub Topics
|
|
132
|
+
- Add ALL relevant topics: `llm`, `router`, `ai`, `gateway`, `open-source`, `nodejs`, `routing`, `llm-routing`, `model-router`
|
|
133
|
+
|
|
134
|
+
### 29. LibHunt (DR 68)
|
|
135
|
+
- **Submit:** https://www.libhunt.com
|
|
136
|
+
- **Tips:** Compare with similar libraries
|
|
137
|
+
|
|
138
|
+
### 30. Openbase (DR 65)
|
|
139
|
+
- **Submit:** https://openbase.com
|
|
140
|
+
|
|
141
|
+
## Tier 4: Long Tail (DR < 50) [10 sites]
|
|
142
|
+
|
|
143
|
+
### 31. Makerlog (DR 45)
|
|
144
|
+
- **Submit:** https://getmakerlog.com
|
|
145
|
+
|
|
146
|
+
### 32. WIP.chat (DR 40)
|
|
147
|
+
- **Submit:** https://wip.chat
|
|
148
|
+
|
|
149
|
+
### 33. Failory (DR 48)
|
|
150
|
+
- **Submit:** https://failory.com/submit
|
|
151
|
+
- **Hook:** "What I learned building an open-source LLM router"
|
|
152
|
+
|
|
153
|
+
### 34. StarterStory (DR 49)
|
|
154
|
+
- **Submit:** https://starterstory.com
|
|
155
|
+
- **Hook:** "From $800/month API bill to $5 — the A3M story"
|
|
156
|
+
|
|
157
|
+
### 35. IndieHackers (DR 56)
|
|
158
|
+
- **Post:** Write a maker story
|
|
159
|
+
- **Hook:** "$800/month → $5 by routing smart. Here's how."
|
|
160
|
+
|
|
161
|
+
### 36. SideProjectors (DR 35)
|
|
162
|
+
- **Submit:** https://sideprojectors.com
|
|
163
|
+
|
|
164
|
+
### 37. Geekflare (DR 54)
|
|
165
|
+
- **Submit:** https://geekflare.com/tools/
|
|
166
|
+
|
|
167
|
+
### 38. YourStory (DR 44)
|
|
168
|
+
- **Submit:** https://yourstory.com/submit
|
|
169
|
+
|
|
170
|
+
### 39. TechCrunch Tip Line
|
|
171
|
+
- **Submit:** tips@techcrunch.com
|
|
172
|
+
- **Hook:** "Open source project beats GPT-4 at 0.5% of the cost"
|
|
173
|
+
|
|
174
|
+
### 40. HackerNoon (DR 53)
|
|
175
|
+
- **Write:** Article as guest writer
|
|
176
|
+
- **Hook:** "I built an open-source LLM router. Here's why you need one."
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## Automation
|
|
181
|
+
|
|
182
|
+
For daily monitoring, set up:
|
|
183
|
+
```
|
|
184
|
+
# Google Alerts
|
|
185
|
+
"LLM routing" "model router" "AI gateway alternative" "LiteLLM alternative"
|
|
186
|
+
|
|
187
|
+
# F5bot — HN keyword monitoring (free tier)
|
|
188
|
+
https://f5bot.com
|
|
189
|
+
|
|
190
|
+
# ReplyGuy — automated reply system
|
|
191
|
+
https://replyguy.com
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## Tracking
|
|
195
|
+
|
|
196
|
+
| # | Site | Submitted | Date | Result |
|
|
197
|
+
|---|------|-----------|------|--------|
|
|
198
|
+
| 1 | Product Hunt | | | |
|
|
199
|
+
| 2 | Hacker News | | | |
|
|
200
|
+
| ... | ... | ... | ... | ... |
|