cli-community-intelligence 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +6523 -0
- package/dist/index.js +844 -0
- package/package.json +17 -0
- package/schema.sql +66 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,844 @@
|
|
|
1
|
+
// src/corpus-db.js
|
|
2
|
+
import Database from "better-sqlite3";
|
|
3
|
+
import { existsSync, mkdirSync, readFileSync } from "fs";
|
|
4
|
+
import { dirname, resolve } from "path";
|
|
5
|
+
import { homedir } from "os";
|
|
6
|
+
import { fileURLToPath } from "url";
|
|
7
|
+
|
|
8
|
+
// ../functional/src/lookup-table.js
|
|
9
|
+
var LookupTable = (items, ItemType, idField = "id") => {
|
|
10
|
+
const addNonEnumerable = (target, key, value) => Object.defineProperty(target, key, { value, enumerable: false });
|
|
11
|
+
const validateTypes = () => {
|
|
12
|
+
if (!ItemType || !ItemType.is) {
|
|
13
|
+
console.error(`You must pass a tagged Type when creating a LookupTable`);
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
if (items.every((item) => ItemType.is(item))) return;
|
|
17
|
+
const found = JSON.stringify(items);
|
|
18
|
+
const w = `Expected each item passed to LookupTable to be a(n) '${ItemType.toString()}'; found ${found}.`;
|
|
19
|
+
console.error(w);
|
|
20
|
+
};
|
|
21
|
+
if (!Array.isArray(items)) items = Object.values(items);
|
|
22
|
+
validateTypes();
|
|
23
|
+
const array = Array.from(items);
|
|
24
|
+
items.forEach((o) => addNonEnumerable(array, o[idField], o));
|
|
25
|
+
addNonEnumerable(array, "ItemType", ItemType);
|
|
26
|
+
addNonEnumerable(array, "idField", idField);
|
|
27
|
+
Object.setPrototypeOf(array, LookupTablePrototype);
|
|
28
|
+
return array;
|
|
29
|
+
};
|
|
30
|
+
LookupTable.is = (o) => !!o?.idField;
|
|
31
|
+
var LookupTablePrototype = Object.create(Array.prototype);
|
|
32
|
+
LookupTablePrototype.get = function(key) {
|
|
33
|
+
return this[key];
|
|
34
|
+
};
|
|
35
|
+
LookupTablePrototype.getById = function(key) {
|
|
36
|
+
return this[key];
|
|
37
|
+
};
|
|
38
|
+
LookupTablePrototype.elementHavingSameIdAsItem = function(item) {
|
|
39
|
+
const id = item[this.idField];
|
|
40
|
+
return this[id];
|
|
41
|
+
};
|
|
42
|
+
LookupTablePrototype.hasItemEqualTo = function(item) {
|
|
43
|
+
return equals_default(this.elementHavingSameIdAsItem(item), item);
|
|
44
|
+
};
|
|
45
|
+
LookupTablePrototype.includesWithId = function(id) {
|
|
46
|
+
return this.some((item) => item.id === id);
|
|
47
|
+
};
|
|
48
|
+
LookupTablePrototype.filter = function(predicate) {
|
|
49
|
+
return LookupTable(Array.prototype.filter.call(this, predicate), this.ItemType, this.idField);
|
|
50
|
+
};
|
|
51
|
+
LookupTablePrototype.sort = function(sortFunc) {
|
|
52
|
+
return LookupTable(Array.prototype.sort.call(Array.from(this), sortFunc), this.ItemType, this.idField);
|
|
53
|
+
};
|
|
54
|
+
LookupTablePrototype.prepend = function(o) {
|
|
55
|
+
return LookupTable([o, ...this], this.ItemType, this.idField);
|
|
56
|
+
};
|
|
57
|
+
LookupTablePrototype.addItem = function(item, sort) {
|
|
58
|
+
if (this.hasItemEqualTo(item)) return this;
|
|
59
|
+
const newItems = [...this, item];
|
|
60
|
+
if (sort) Array.prototype.sort.call(newItems, sort);
|
|
61
|
+
return LookupTable(newItems, this.ItemType, this.idField);
|
|
62
|
+
};
|
|
63
|
+
LookupTablePrototype.addItemWithId = function(item, sort) {
|
|
64
|
+
if (this.hasItemEqualTo(item)) return this;
|
|
65
|
+
const oldItem = this.elementHavingSameIdAsItem(item);
|
|
66
|
+
let newItems;
|
|
67
|
+
if (oldItem) newItems = this.map((existingItem) => existingItem === oldItem ? item : existingItem);
|
|
68
|
+
else newItems = [...this, item];
|
|
69
|
+
if (sort) Array.prototype.sort.call(newItems, sort);
|
|
70
|
+
return LookupTable(newItems, this.ItemType, this.idField);
|
|
71
|
+
};
|
|
72
|
+
LookupTablePrototype.removeItem = function(item) {
|
|
73
|
+
if (!this.elementHavingSameIdAsItem(item)) return this;
|
|
74
|
+
return LookupTable(without_default(item, this), this.ItemType, this.idField);
|
|
75
|
+
};
|
|
76
|
+
LookupTablePrototype.removeItemWithId = function(id) {
|
|
77
|
+
const item = this[id];
|
|
78
|
+
if (!item) return this;
|
|
79
|
+
return LookupTable(without_default(item, this), this.ItemType, this.idField);
|
|
80
|
+
};
|
|
81
|
+
LookupTablePrototype.toggleItem = function(item, sort) {
|
|
82
|
+
const hasItem = this.elementHavingSameIdAsItem(item) === item;
|
|
83
|
+
return hasItem ? this.removeItem(item) : this.addItem(item, sort);
|
|
84
|
+
};
|
|
85
|
+
LookupTablePrototype.pick = function(ids) {
|
|
86
|
+
const subset = ids.map((id) => this[id]);
|
|
87
|
+
return LookupTable(subset, this.ItemType, this.idField);
|
|
88
|
+
};
|
|
89
|
+
LookupTablePrototype.moveElement = function(fromIndex, toIndex) {
|
|
90
|
+
if (fromIndex < 0 || fromIndex >= this.length || toIndex < 0 || toIndex >= this.length) return this;
|
|
91
|
+
const newArray = [...this];
|
|
92
|
+
const [item] = newArray.splice(fromIndex, 1);
|
|
93
|
+
newArray.splice(toIndex, 0, item);
|
|
94
|
+
return LookupTable(newArray, this.ItemType, this.idField);
|
|
95
|
+
};
|
|
96
|
+
LookupTablePrototype.updateAll = function(fn) {
|
|
97
|
+
return LookupTable(Array.prototype.map.call(this, fn), this.ItemType, this.idField);
|
|
98
|
+
};
|
|
99
|
+
LookupTablePrototype.updateWhere = function(predicate, fn) {
|
|
100
|
+
const updated = Array.prototype.map.call(this, (item) => predicate(item) ? fn(item) : item);
|
|
101
|
+
return LookupTable(updated, this.ItemType, this.idField);
|
|
102
|
+
};
|
|
103
|
+
LookupTablePrototype.idForItem = function(item) {
|
|
104
|
+
return item[this.idField];
|
|
105
|
+
};
|
|
106
|
+
var lookup_table_default = LookupTable;
|
|
107
|
+
|
|
108
|
+
// ../functional/src/ramda-like/keys.js
|
|
109
|
+
var keys = (obj) => Object(obj) !== obj ? [] : Object.keys(obj);
|
|
110
|
+
var keys_default = keys;
|
|
111
|
+
|
|
112
|
+
// ../functional/src/ramda-like/type.js
|
|
113
|
+
var type = (val) => val === null ? "Null" : val === void 0 ? "Undefined" : Object.prototype.toString.call(val).slice(8, -1);
|
|
114
|
+
var type_default = type;
|
|
115
|
+
|
|
116
|
+
// ../functional/src/ramda-like/equals.js
|
|
117
|
+
var _has = (prop, obj) => Object.prototype.hasOwnProperty.call(obj, prop);
|
|
118
|
+
var _functionName = (f) => {
|
|
119
|
+
const match2 = String(f).match(/^function (\w*)/);
|
|
120
|
+
return match2 == null ? "" : match2[1];
|
|
121
|
+
};
|
|
122
|
+
var _arrayFromIterator = (iterator) => {
|
|
123
|
+
const result = [];
|
|
124
|
+
let next;
|
|
125
|
+
while (!(next = iterator.next()).done) result.push(next.value);
|
|
126
|
+
return result;
|
|
127
|
+
};
|
|
128
|
+
var _includesWith = (f, x, list) => {
|
|
129
|
+
const len = list.length;
|
|
130
|
+
for (let i = 0; i < len; i++) if (f(x, list[i])) return true;
|
|
131
|
+
return false;
|
|
132
|
+
};
|
|
133
|
+
var _uniqContentEquals = (aIterator, bIterator, stackA, stackB) => {
|
|
134
|
+
const a = _arrayFromIterator(aIterator);
|
|
135
|
+
const b = _arrayFromIterator(bIterator);
|
|
136
|
+
const eq = (_a, _b) => _equals(_a, _b, stackA.slice(), stackB.slice());
|
|
137
|
+
return !_includesWith((b2, aItem) => !_includesWith(eq, aItem, b2), b, a);
|
|
138
|
+
};
|
|
139
|
+
function _equals(a, b, stackA, stackB) {
|
|
140
|
+
if (Object.is(a, b)) return true;
|
|
141
|
+
const typeA = type_default(a);
|
|
142
|
+
if (typeA !== type_default(b)) return false;
|
|
143
|
+
if (a == null || b == null) return false;
|
|
144
|
+
if (typeof a.equals === "function" || typeof b.equals === "function")
|
|
145
|
+
return typeof a.equals === "function" && a.equals(b) && typeof b.equals === "function" && b.equals(a);
|
|
146
|
+
switch (typeA) {
|
|
147
|
+
case "Arguments":
|
|
148
|
+
case "Array":
|
|
149
|
+
case "Object":
|
|
150
|
+
if (typeof a.constructor === "function" && _functionName(a.constructor) === "Promise") return a === b;
|
|
151
|
+
break;
|
|
152
|
+
case "Boolean":
|
|
153
|
+
case "Number":
|
|
154
|
+
case "String":
|
|
155
|
+
if (!(typeof a === typeof b && Object.is(a.valueOf(), b.valueOf()))) return false;
|
|
156
|
+
break;
|
|
157
|
+
case "Date":
|
|
158
|
+
if (!Object.is(a.valueOf(), b.valueOf())) return false;
|
|
159
|
+
break;
|
|
160
|
+
case "Error":
|
|
161
|
+
return a.name === b.name && a.message === b.message;
|
|
162
|
+
case "RegExp":
|
|
163
|
+
if (!(a.source === b.source && a.global === b.global && a.ignoreCase === b.ignoreCase && a.multiline === b.multiline && a.sticky === b.sticky && a.unicode === b.unicode))
|
|
164
|
+
return false;
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
for (let i = stackA.length - 1; i >= 0; i--) if (stackA[i] === a) return stackB[i] === b;
|
|
168
|
+
switch (typeA) {
|
|
169
|
+
case "Map":
|
|
170
|
+
if (a.size !== b.size) return false;
|
|
171
|
+
return _uniqContentEquals(a.entries(), b.entries(), stackA.concat([a]), stackB.concat([b]));
|
|
172
|
+
case "Set":
|
|
173
|
+
if (a.size !== b.size) return false;
|
|
174
|
+
return _uniqContentEquals(a.values(), b.values(), stackA.concat([a]), stackB.concat([b]));
|
|
175
|
+
case "Arguments":
|
|
176
|
+
case "Array":
|
|
177
|
+
case "Object":
|
|
178
|
+
case "Boolean":
|
|
179
|
+
case "Number":
|
|
180
|
+
case "String":
|
|
181
|
+
case "Date":
|
|
182
|
+
case "Error":
|
|
183
|
+
case "RegExp":
|
|
184
|
+
case "Int8Array":
|
|
185
|
+
case "Uint8Array":
|
|
186
|
+
case "Uint8ClampedArray":
|
|
187
|
+
case "Int16Array":
|
|
188
|
+
case "Uint16Array":
|
|
189
|
+
case "Int32Array":
|
|
190
|
+
case "Uint32Array":
|
|
191
|
+
case "Float32Array":
|
|
192
|
+
case "Float64Array":
|
|
193
|
+
case "ArrayBuffer":
|
|
194
|
+
break;
|
|
195
|
+
default:
|
|
196
|
+
return false;
|
|
197
|
+
}
|
|
198
|
+
const keysA = keys_default(a);
|
|
199
|
+
if (keysA.length !== keys_default(b).length) return false;
|
|
200
|
+
const extendedStackA = stackA.concat([a]);
|
|
201
|
+
const extendedStackB = stackB.concat([b]);
|
|
202
|
+
for (let i = keysA.length - 1; i >= 0; i--) {
|
|
203
|
+
const key = keysA[i];
|
|
204
|
+
if (!(_has(key, b) && _equals(b[key], a[key], extendedStackA, extendedStackB))) return false;
|
|
205
|
+
}
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
var equals = (a, b) => _equals(a, b, [], []);
|
|
209
|
+
var equals_default = equals;
|
|
210
|
+
|
|
211
|
+
// ../functional/src/ramda-like/pick.js
|
|
212
|
+
var pick = function pick2(names, o) {
|
|
213
|
+
const result = {};
|
|
214
|
+
for (let i = 0; i < names.length; i++) {
|
|
215
|
+
const name = names[i];
|
|
216
|
+
if (name in o) result[name] = o[name];
|
|
217
|
+
}
|
|
218
|
+
return result;
|
|
219
|
+
};
|
|
220
|
+
var pick_default = pick;
|
|
221
|
+
|
|
222
|
+
// ../functional/src/ramda-like/memoize-redux-state.js
|
|
223
|
+
var memoizeReduxState = (keys2, f) => {
|
|
224
|
+
const stateMatches = (state) => {
|
|
225
|
+
if (keys2.length === 0) return state === previousState;
|
|
226
|
+
return keys2.every((key) => state[key] === previousState[key]);
|
|
227
|
+
};
|
|
228
|
+
let previousState = {};
|
|
229
|
+
let previousArgsStringified;
|
|
230
|
+
let previousValue;
|
|
231
|
+
return (state, ...args) => {
|
|
232
|
+
const newArgsStringified = JSON.stringify(args);
|
|
233
|
+
if (stateMatches(state) && newArgsStringified === previousArgsStringified) return previousValue;
|
|
234
|
+
previousState = pick_default(keys2, state);
|
|
235
|
+
previousArgsStringified = newArgsStringified;
|
|
236
|
+
previousValue = f(state, ...args);
|
|
237
|
+
return previousValue;
|
|
238
|
+
};
|
|
239
|
+
};
|
|
240
|
+
var memoizeReduxStatePerKey = (globalKeys, keyedStateKey, f) => {
|
|
241
|
+
const globalStateChanged = (state) => (
|
|
242
|
+
// eslint-disable-next-line no-restricted-syntax -- memoizer must access state directly
|
|
243
|
+
globalKeys.some((key) => state[key] !== previousGlobalState[key])
|
|
244
|
+
);
|
|
245
|
+
const cacheByKey = /* @__PURE__ */ new Map();
|
|
246
|
+
let previousGlobalState = {};
|
|
247
|
+
return (state, key, ...rest) => {
|
|
248
|
+
const keyedState = state[keyedStateKey];
|
|
249
|
+
const keyedValue = keyedState?.get ? keyedState.get(key) : keyedState?.[key];
|
|
250
|
+
if (globalStateChanged(state)) {
|
|
251
|
+
cacheByKey.clear();
|
|
252
|
+
previousGlobalState = pick_default(globalKeys, state);
|
|
253
|
+
}
|
|
254
|
+
const cached = cacheByKey.get(key);
|
|
255
|
+
const { keyedValue: cachedKeyed, restStringified: cachedRest, value: cachedValue } = cached || {};
|
|
256
|
+
const cheapHit = cached && cachedKeyed === keyedValue;
|
|
257
|
+
const restMatch = cheapHit && (rest.length > 0 ? JSON.stringify(rest) : "") === cachedRest;
|
|
258
|
+
if (restMatch) return cachedValue;
|
|
259
|
+
const value = f(state, key, ...rest);
|
|
260
|
+
const restStringified = rest.length > 0 ? JSON.stringify(rest) : "";
|
|
261
|
+
cacheByKey.set(key, { keyedValue, restStringified, value });
|
|
262
|
+
return value;
|
|
263
|
+
};
|
|
264
|
+
};
|
|
265
|
+
var MemoizeReduxState = { memoizeReduxState, memoizeReduxStatePerKey };
|
|
266
|
+
|
|
267
|
+
// ../functional/src/ramda-like/list.js
|
|
268
|
+
var reject = (f, a) => a.filter((a2) => !f(a2));
|
|
269
|
+
|
|
270
|
+
// ../functional/src/ramda-like/without.js
|
|
271
|
+
var without = (toBeRemoved, list) => {
|
|
272
|
+
if (!Array.isArray(toBeRemoved)) toBeRemoved = [toBeRemoved];
|
|
273
|
+
return reject((o) => toBeRemoved.includes(o), list);
|
|
274
|
+
};
|
|
275
|
+
var without_default = without;
|
|
276
|
+
|
|
277
|
+
// ../functional/index.js
|
|
278
|
+
var { memoizeReduxState: memoizeReduxState2, memoizeReduxStatePerKey: memoizeReduxStatePerKey2 } = MemoizeReduxState;
|
|
279
|
+
|
|
280
|
+
// ../cli-type-generator/runtime-for-generated-types.js
|
|
281
|
+
var match = (tagNames) => {
|
|
282
|
+
const validateVariants = (variants) => {
|
|
283
|
+
const missing = tagNames.find((variant) => !variants[variant]);
|
|
284
|
+
if (missing) throw new TypeError(`Constructors given to match didn't include: ${missing}`);
|
|
285
|
+
};
|
|
286
|
+
return function(variants) {
|
|
287
|
+
validateVariants(variants);
|
|
288
|
+
const variant = variants[this["@@tagName"]];
|
|
289
|
+
return variant.call(variants, this);
|
|
290
|
+
};
|
|
291
|
+
};
|
|
292
|
+
var _toString = (value) => {
|
|
293
|
+
if (typeof value === "undefined") return "undefined";
|
|
294
|
+
if (value === null) return "null";
|
|
295
|
+
if (value["@@typeName"]) return value.toString();
|
|
296
|
+
if (Array.isArray(value)) return "[" + value.map((item) => _toString(item)).join(", ") + "]";
|
|
297
|
+
if (typeof value === "string") return '"' + value + '"';
|
|
298
|
+
if (value instanceof Date) return value.toISOString();
|
|
299
|
+
if (value instanceof RegExp) return value.toString();
|
|
300
|
+
if (typeof value === "object") return JSON.stringify(value);
|
|
301
|
+
return value;
|
|
302
|
+
};
|
|
303
|
+
var lookupTableToFirestore = (Type, idField, encodeTimestamps, lookupTable) => {
|
|
304
|
+
const itemToFirestoreEntry = (item, index) => [
|
|
305
|
+
item[idField],
|
|
306
|
+
{ ...Type.toFirestore(item, encodeTimestamps), _order: index }
|
|
307
|
+
];
|
|
308
|
+
return Object.fromEntries(lookupTable.map(itemToFirestoreEntry));
|
|
309
|
+
};
|
|
310
|
+
var lookupTableFromFirestore = (Type, idField, decodeTimestamps, o) => {
|
|
311
|
+
const firestoreItemToDomain = (item) => {
|
|
312
|
+
const { _order, ...rest } = item;
|
|
313
|
+
return Type.fromFirestore(rest, decodeTimestamps);
|
|
314
|
+
};
|
|
315
|
+
return lookup_table_default(
|
|
316
|
+
Object.values(o || {}).sort((a, b) => (a._order ?? 0) - (b._order ?? 0)).map(firestoreItemToDomain),
|
|
317
|
+
Type,
|
|
318
|
+
idField
|
|
319
|
+
);
|
|
320
|
+
};
|
|
321
|
+
var validateArgumentLength = (constructorName, expectedCount, args) => {
|
|
322
|
+
if (args.length === expectedCount) return;
|
|
323
|
+
debugger;
|
|
324
|
+
const message = `In constructor ${constructorName}: expected ${expectedCount} arguments, found ${args.length}`;
|
|
325
|
+
throw new TypeError(message);
|
|
326
|
+
};
|
|
327
|
+
var validateRegex = (constructorName, regex, field, optional, s) => {
|
|
328
|
+
validateString(constructorName, field, optional, s);
|
|
329
|
+
if (optional && (s === void 0 || s === null)) return;
|
|
330
|
+
if (s.match(regex)) return;
|
|
331
|
+
debugger;
|
|
332
|
+
const message = `In constructor ${constructorName}: expected ${field} to match ${regex}; found ${_toString(s)})`;
|
|
333
|
+
throw new TypeError(message);
|
|
334
|
+
};
|
|
335
|
+
var validateNumber = (constructorName, field, optional, n) => {
|
|
336
|
+
if (optional && (n === void 0 || n === null)) return;
|
|
337
|
+
if (typeof n === "number") return;
|
|
338
|
+
debugger;
|
|
339
|
+
const message = `In constructor ${constructorName}: expected ${field} to have type Number; found ${_toString(n)})`;
|
|
340
|
+
throw new TypeError(message);
|
|
341
|
+
};
|
|
342
|
+
var validateString = (constructorName, field, optional, s) => {
|
|
343
|
+
if (optional && (s === void 0 || s === null)) return;
|
|
344
|
+
if (typeof s === "string") return;
|
|
345
|
+
debugger;
|
|
346
|
+
const message = `In constructor ${constructorName}: expected ${field} to have type String; found ${_toString(s)})`;
|
|
347
|
+
throw new TypeError(message);
|
|
348
|
+
};
|
|
349
|
+
var validateObject = (constructorName, field, optional, o) => {
|
|
350
|
+
if (optional && (o === void 0 || o === null)) return;
|
|
351
|
+
if (typeof o === "object" && o !== null) return;
|
|
352
|
+
debugger;
|
|
353
|
+
const message = `In constructor ${constructorName}: expected ${field} to have type Object; found ${_toString(o)})`;
|
|
354
|
+
throw new TypeError(message);
|
|
355
|
+
};
|
|
356
|
+
var validateDate = (constructorName, field, optional, d) => {
|
|
357
|
+
if (optional && (d === void 0 || d === null)) return;
|
|
358
|
+
if (d instanceof Date) return;
|
|
359
|
+
debugger;
|
|
360
|
+
const message = `In constructor ${constructorName}: expected ${field} to have type Date; found ${_toString(d)})`;
|
|
361
|
+
throw new TypeError(message);
|
|
362
|
+
};
|
|
363
|
+
var validateBoolean = (constructorName, field, optional, b) => {
|
|
364
|
+
if (optional && (b === void 0 || b === null)) return;
|
|
365
|
+
if (typeof b === "boolean") return;
|
|
366
|
+
debugger;
|
|
367
|
+
const message = `In constructor ${constructorName}: expected ${field} to have type Boolean; found ${_toString(b)})`;
|
|
368
|
+
throw new TypeError(message);
|
|
369
|
+
};
|
|
370
|
+
var validateTag = (constructorName, expectedType, field, optional, o) => {
|
|
371
|
+
if (optional && (o === void 0 || o === null)) return;
|
|
372
|
+
if (o?.["@@typeName"] === expectedType) return;
|
|
373
|
+
debugger;
|
|
374
|
+
const found = _toString(o);
|
|
375
|
+
const message = `In constructor ${constructorName}: expected ${field} to have type ${expectedType}; found ${found}`;
|
|
376
|
+
throw new TypeError(message);
|
|
377
|
+
};
|
|
378
|
+
var validateArray = (constructorName, arrayDepth, baseType, taggedType, field, optional, a) => {
|
|
379
|
+
const repeatCharacter = (char, n) => Array(n).fill(char).join("");
|
|
380
|
+
const buildNestedTypeString = () => repeatCharacter("[", arrayDepth) + (taggedType ?? baseType) + repeatCharacter("]", arrayDepth);
|
|
381
|
+
const checkArrayAtDepth = (value, currentDepth) => {
|
|
382
|
+
if (!Array.isArray(value)) return { valid: false, element: void 0 };
|
|
383
|
+
if (value.length === 0) return { valid: true, element: void 0, empty: true };
|
|
384
|
+
if (currentDepth + 1 >= arrayDepth) return { valid: true, element: value[0] };
|
|
385
|
+
return checkArrayAtDepth(value[0], currentDepth + 1);
|
|
386
|
+
};
|
|
387
|
+
const isValidBaseType = (element2) => {
|
|
388
|
+
if (baseType === "String") return typeof element2 === "string";
|
|
389
|
+
if (baseType === "Number") return typeof element2 === "number";
|
|
390
|
+
if (baseType === "Object") return typeof element2 === "object";
|
|
391
|
+
if (baseType === "Date") return element2 instanceof Date;
|
|
392
|
+
if (baseType === "Any") return true;
|
|
393
|
+
if (baseType === "Tagged") return element2?.["@@typeName"] === taggedType;
|
|
394
|
+
return false;
|
|
395
|
+
};
|
|
396
|
+
if (optional && (a === void 0 || a === null)) return;
|
|
397
|
+
const { valid, empty, element } = checkArrayAtDepth(a, 0);
|
|
398
|
+
const nestedType = buildNestedTypeString();
|
|
399
|
+
const found = _toString(a);
|
|
400
|
+
const prefix = `In constructor ${constructorName}`;
|
|
401
|
+
const expected = `expected ${field} to have type ${nestedType}`;
|
|
402
|
+
if (!valid) {
|
|
403
|
+
debugger;
|
|
404
|
+
throw new TypeError(`${prefix}: ${expected}; found ${found}`);
|
|
405
|
+
}
|
|
406
|
+
if (empty) return;
|
|
407
|
+
if (isValidBaseType(element)) return;
|
|
408
|
+
debugger;
|
|
409
|
+
throw new TypeError(`${prefix}: ${expected}; found ${found}`);
|
|
410
|
+
};
|
|
411
|
+
var validateLookupTable = (constructorName, expectedItemType, field, optional, lt) => {
|
|
412
|
+
if (optional && (lt === void 0 || lt === null)) return;
|
|
413
|
+
if (!lt || typeof lt !== "object" || !lt.idField) {
|
|
414
|
+
debugger;
|
|
415
|
+
const found = _toString(lt);
|
|
416
|
+
const message = `In constructor ${constructorName}: expected ${field} to be a LookupTable; found ${found}`;
|
|
417
|
+
throw new TypeError(message);
|
|
418
|
+
}
|
|
419
|
+
if (lt.length === 0) return;
|
|
420
|
+
const firstItem = lt[0];
|
|
421
|
+
if (firstItem?.["@@typeName"] === expectedItemType) return;
|
|
422
|
+
debugger;
|
|
423
|
+
const actualType = firstItem?.["@@typeName"] || "unknown";
|
|
424
|
+
const expected = `a LookupTable<${expectedItemType}>`;
|
|
425
|
+
const actual = `LookupTable<${actualType}>`;
|
|
426
|
+
throw new TypeError(`In constructor ${constructorName}: expected ${field} to be ${expected}; found ${actual}`);
|
|
427
|
+
};
|
|
428
|
+
var piiRedactions = {
|
|
429
|
+
email: (v) => v.replace(/(.).*(@.*)/, "$1***$2"),
|
|
430
|
+
displayName: (v) => v.replace(/\b(\w)\w*/g, "$1***"),
|
|
431
|
+
phoneNumber: (v) => v.replace(/(\+?\d{1,3}[-.\s]?)?(\(?\d{3}\)?[-.\s]?)(\d{3}[-.\s]?)(\d{4})/, "***-***-$4")
|
|
432
|
+
};
|
|
433
|
+
var redact = (value) => {
|
|
434
|
+
const hasPii = (v) => {
|
|
435
|
+
const fieldHasPii = (fieldValue) => {
|
|
436
|
+
if (Array.isArray(fieldValue)) return fieldValue.some(hasPii);
|
|
437
|
+
if (typeof fieldValue === "object") return hasPii(fieldValue);
|
|
438
|
+
return false;
|
|
439
|
+
};
|
|
440
|
+
if (v === void 0 || v === null) return false;
|
|
441
|
+
const hasDirectPii = Object.keys(piiRedactions).some((field) => v[field] && typeof v[field] === "string");
|
|
442
|
+
if (hasDirectPii) return true;
|
|
443
|
+
return Object.values(v).some(fieldHasPii);
|
|
444
|
+
};
|
|
445
|
+
const redactField = (k, v) => {
|
|
446
|
+
if (piiRedactions[k] && typeof v === "string") return piiRedactions[k](v);
|
|
447
|
+
if (v?.["@@typeName"]) return redact(v);
|
|
448
|
+
if (lookup_table_default.is(v)) return lookup_table_default(v.map(redact), v.ItemType, v.idField);
|
|
449
|
+
if (Array.isArray(v) && v[0]?.["@@typeName"]) return v.map(redact);
|
|
450
|
+
return v;
|
|
451
|
+
};
|
|
452
|
+
const markAsRedacted = (obj) => {
|
|
453
|
+
Object.defineProperty(obj, "__redacted", {
|
|
454
|
+
value: true,
|
|
455
|
+
enumerable: false,
|
|
456
|
+
writable: false,
|
|
457
|
+
configurable: false
|
|
458
|
+
});
|
|
459
|
+
return obj;
|
|
460
|
+
};
|
|
461
|
+
if (value?.__redacted) return value;
|
|
462
|
+
if (!value?.["@@typeName"]) return value;
|
|
463
|
+
if (!hasPii(value)) return markAsRedacted(value);
|
|
464
|
+
const redactedObject = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, redactField(k, v)]));
|
|
465
|
+
const constructor = Object.getPrototypeOf(value).constructor;
|
|
466
|
+
return markAsRedacted(constructor.from(redactedObject));
|
|
467
|
+
};
|
|
468
|
+
var parseTaggedValue = (plain, variantMap) => {
|
|
469
|
+
if (plain === void 0 || plain === null) return plain;
|
|
470
|
+
const tagName = plain["@@tagName"];
|
|
471
|
+
if (!tagName) throw new TypeError(`parseTaggedValue: missing @@tagName on ${_toString(plain)}`);
|
|
472
|
+
const Constructor = variantMap[tagName];
|
|
473
|
+
if (!Constructor)
|
|
474
|
+
throw new TypeError(
|
|
475
|
+
`parseTaggedValue: unknown variant '${tagName}' (known: ${Object.keys(variantMap).join(", ")})`
|
|
476
|
+
);
|
|
477
|
+
return Constructor._from(plain);
|
|
478
|
+
};
|
|
479
|
+
var RuntimeForGeneratedTypes = {
|
|
480
|
+
validateArgumentLength,
|
|
481
|
+
validateArray,
|
|
482
|
+
validateBoolean,
|
|
483
|
+
validateDate,
|
|
484
|
+
validateNumber,
|
|
485
|
+
validateObject,
|
|
486
|
+
validateString,
|
|
487
|
+
validateTag,
|
|
488
|
+
validateRegex,
|
|
489
|
+
validateLookupTable,
|
|
490
|
+
lookupTableToFirestore,
|
|
491
|
+
lookupTableFromFirestore,
|
|
492
|
+
match,
|
|
493
|
+
_toString,
|
|
494
|
+
redact,
|
|
495
|
+
parseTaggedValue
|
|
496
|
+
};
|
|
497
|
+
|
|
498
|
+
// src/types/post.js
|
|
499
|
+
var Post = function Post2(source, sourceId, channel, author, title, body, parentId, url, score, createdAt, postType) {
|
|
500
|
+
const constructorName = "Post(source, sourceId, channel, author, title, body, parentId, url, score, createdAt, postType)";
|
|
501
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "source", false, source);
|
|
502
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "sourceId", false, sourceId);
|
|
503
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "channel", false, channel);
|
|
504
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "author", true, author);
|
|
505
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "title", true, title);
|
|
506
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "body", false, body);
|
|
507
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "parentId", true, parentId);
|
|
508
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "url", true, url);
|
|
509
|
+
RuntimeForGeneratedTypes.validateNumber(constructorName, "score", true, score);
|
|
510
|
+
RuntimeForGeneratedTypes.validateString(constructorName, "createdAt", false, createdAt);
|
|
511
|
+
RuntimeForGeneratedTypes.validateRegex(constructorName, /^(post|comment)$/, "postType", false, postType);
|
|
512
|
+
const result = Object.create(prototype);
|
|
513
|
+
result.source = source;
|
|
514
|
+
result.sourceId = sourceId;
|
|
515
|
+
result.channel = channel;
|
|
516
|
+
if (author !== void 0) result.author = author;
|
|
517
|
+
if (title !== void 0) result.title = title;
|
|
518
|
+
result.body = body;
|
|
519
|
+
if (parentId !== void 0) result.parentId = parentId;
|
|
520
|
+
if (url !== void 0) result.url = url;
|
|
521
|
+
if (score !== void 0) result.score = score;
|
|
522
|
+
result.createdAt = createdAt;
|
|
523
|
+
result.postType = postType;
|
|
524
|
+
return result;
|
|
525
|
+
};
|
|
526
|
+
var postToString = function() {
|
|
527
|
+
return `Post(${RuntimeForGeneratedTypes._toString(this.source)},
|
|
528
|
+
${RuntimeForGeneratedTypes._toString(this.sourceId)},
|
|
529
|
+
${RuntimeForGeneratedTypes._toString(this.channel)},
|
|
530
|
+
${RuntimeForGeneratedTypes._toString(this.author)},
|
|
531
|
+
${RuntimeForGeneratedTypes._toString(this.title)},
|
|
532
|
+
${RuntimeForGeneratedTypes._toString(this.body)},
|
|
533
|
+
${RuntimeForGeneratedTypes._toString(this.parentId)},
|
|
534
|
+
${RuntimeForGeneratedTypes._toString(this.url)},
|
|
535
|
+
${RuntimeForGeneratedTypes._toString(this.score)},
|
|
536
|
+
${RuntimeForGeneratedTypes._toString(this.createdAt)},
|
|
537
|
+
${RuntimeForGeneratedTypes._toString(this.postType)})`;
|
|
538
|
+
};
|
|
539
|
+
var postToJSON = function() {
|
|
540
|
+
return this;
|
|
541
|
+
};
|
|
542
|
+
var prototype = Object.create(Object.prototype, {
|
|
543
|
+
"@@typeName": { value: "Post", enumerable: false },
|
|
544
|
+
toString: { value: postToString, enumerable: false },
|
|
545
|
+
toJSON: { value: postToJSON, enumerable: false },
|
|
546
|
+
constructor: { value: Post, enumerable: false, writable: true, configurable: true }
|
|
547
|
+
});
|
|
548
|
+
Post.prototype = prototype;
|
|
549
|
+
Post.toString = () => "Post";
|
|
550
|
+
Post.is = (v) => v && v["@@typeName"] === "Post";
|
|
551
|
+
Post._from = (_input) => {
|
|
552
|
+
const { source, sourceId, channel, author, title, body, parentId, url, score, createdAt, postType } = _input;
|
|
553
|
+
return Post(source, sourceId, channel, author, title, body, parentId, url, score, createdAt, postType);
|
|
554
|
+
};
|
|
555
|
+
Post.from = Post._from;
|
|
556
|
+
Post.fromJSON = (json) => json == null ? json : Post._from(json);
|
|
557
|
+
|
|
558
|
+
// src/types/revisit-reason.js
|
|
559
|
+
var RevisitReason = { toString: () => "RevisitReason" };
|
|
560
|
+
Object.defineProperty(RevisitReason, "@@typeName", { value: "RevisitReason", enumerable: false });
|
|
561
|
+
Object.defineProperty(RevisitReason, "@@tagNames", { value: ["Recent", "HighScore", "HasMore"], enumerable: false });
|
|
562
|
+
var RevisitReasonPrototype = {};
|
|
563
|
+
Object.defineProperty(RevisitReasonPrototype, "match", {
|
|
564
|
+
value: RuntimeForGeneratedTypes.match(RevisitReason["@@tagNames"]),
|
|
565
|
+
enumerable: false
|
|
566
|
+
});
|
|
567
|
+
Object.defineProperty(RevisitReasonPrototype, "constructor", {
|
|
568
|
+
value: RevisitReason,
|
|
569
|
+
enumerable: false,
|
|
570
|
+
writable: true,
|
|
571
|
+
configurable: true
|
|
572
|
+
});
|
|
573
|
+
RevisitReason.prototype = RevisitReasonPrototype;
|
|
574
|
+
var toString = {
|
|
575
|
+
recent: function() {
|
|
576
|
+
return `RevisitReason.Recent()`;
|
|
577
|
+
},
|
|
578
|
+
highScore: function() {
|
|
579
|
+
return `RevisitReason.HighScore()`;
|
|
580
|
+
},
|
|
581
|
+
hasMore: function() {
|
|
582
|
+
return `RevisitReason.HasMore()`;
|
|
583
|
+
}
|
|
584
|
+
};
|
|
585
|
+
var toJSON = {
|
|
586
|
+
recent: function() {
|
|
587
|
+
return Object.assign({ "@@tagName": this["@@tagName"] }, this);
|
|
588
|
+
},
|
|
589
|
+
highScore: function() {
|
|
590
|
+
return Object.assign({ "@@tagName": this["@@tagName"] }, this);
|
|
591
|
+
},
|
|
592
|
+
hasMore: function() {
|
|
593
|
+
return Object.assign({ "@@tagName": this["@@tagName"] }, this);
|
|
594
|
+
}
|
|
595
|
+
};
|
|
596
|
+
var RecentConstructor = function Recent() {
|
|
597
|
+
const constructorName = "RevisitReason.Recent()";
|
|
598
|
+
RuntimeForGeneratedTypes.validateArgumentLength(constructorName, 0, arguments);
|
|
599
|
+
const result = Object.create(RecentPrototype);
|
|
600
|
+
return result;
|
|
601
|
+
};
|
|
602
|
+
RevisitReason.Recent = RecentConstructor;
|
|
603
|
+
var HighScoreConstructor = function HighScore() {
|
|
604
|
+
const constructorName = "RevisitReason.HighScore()";
|
|
605
|
+
RuntimeForGeneratedTypes.validateArgumentLength(constructorName, 0, arguments);
|
|
606
|
+
const result = Object.create(HighScorePrototype);
|
|
607
|
+
return result;
|
|
608
|
+
};
|
|
609
|
+
RevisitReason.HighScore = HighScoreConstructor;
|
|
610
|
+
var HasMoreConstructor = function HasMore() {
|
|
611
|
+
const constructorName = "RevisitReason.HasMore()";
|
|
612
|
+
RuntimeForGeneratedTypes.validateArgumentLength(constructorName, 0, arguments);
|
|
613
|
+
const result = Object.create(HasMorePrototype);
|
|
614
|
+
return result;
|
|
615
|
+
};
|
|
616
|
+
RevisitReason.HasMore = HasMoreConstructor;
|
|
617
|
+
var RecentPrototype = Object.create(RevisitReasonPrototype, {
|
|
618
|
+
"@@tagName": { value: "Recent", enumerable: false },
|
|
619
|
+
"@@typeName": { value: "RevisitReason", enumerable: false },
|
|
620
|
+
toString: { value: toString.recent, enumerable: false },
|
|
621
|
+
toJSON: { value: toJSON.recent, enumerable: false },
|
|
622
|
+
constructor: { value: RecentConstructor, enumerable: false, writable: true, configurable: true }
|
|
623
|
+
});
|
|
624
|
+
var HighScorePrototype = Object.create(RevisitReasonPrototype, {
|
|
625
|
+
"@@tagName": { value: "HighScore", enumerable: false },
|
|
626
|
+
"@@typeName": { value: "RevisitReason", enumerable: false },
|
|
627
|
+
toString: { value: toString.highScore, enumerable: false },
|
|
628
|
+
toJSON: { value: toJSON.highScore, enumerable: false },
|
|
629
|
+
constructor: { value: HighScoreConstructor, enumerable: false, writable: true, configurable: true }
|
|
630
|
+
});
|
|
631
|
+
var HasMorePrototype = Object.create(RevisitReasonPrototype, {
|
|
632
|
+
"@@tagName": { value: "HasMore", enumerable: false },
|
|
633
|
+
"@@typeName": { value: "RevisitReason", enumerable: false },
|
|
634
|
+
toString: { value: toString.hasMore, enumerable: false },
|
|
635
|
+
toJSON: { value: toJSON.hasMore, enumerable: false },
|
|
636
|
+
constructor: { value: HasMoreConstructor, enumerable: false, writable: true, configurable: true }
|
|
637
|
+
});
|
|
638
|
+
RecentConstructor.prototype = RecentPrototype;
|
|
639
|
+
HighScoreConstructor.prototype = HighScorePrototype;
|
|
640
|
+
HasMoreConstructor.prototype = HasMorePrototype;
|
|
641
|
+
RecentConstructor.is = (val) => val != null && typeof val === "object" && val["@@tagName"] === "Recent";
|
|
642
|
+
HighScoreConstructor.is = (val) => val != null && typeof val === "object" && val["@@tagName"] === "HighScore";
|
|
643
|
+
HasMoreConstructor.is = (val) => val != null && typeof val === "object" && val["@@tagName"] === "HasMore";
|
|
644
|
+
RecentConstructor.toString = () => "RevisitReason.Recent";
|
|
645
|
+
HighScoreConstructor.toString = () => "RevisitReason.HighScore";
|
|
646
|
+
HasMoreConstructor.toString = () => "RevisitReason.HasMore";
|
|
647
|
+
RecentConstructor._from = (_input) => RevisitReason.Recent();
|
|
648
|
+
HighScoreConstructor._from = (_input) => RevisitReason.HighScore();
|
|
649
|
+
HasMoreConstructor._from = (_input) => RevisitReason.HasMore();
|
|
650
|
+
RecentConstructor.from = RecentConstructor._from;
|
|
651
|
+
HighScoreConstructor.from = HighScoreConstructor._from;
|
|
652
|
+
HasMoreConstructor.from = HasMoreConstructor._from;
|
|
653
|
+
RevisitReason.is = (v) => v != null && typeof v === "object" && v["@@typeName"] === "RevisitReason";
|
|
654
|
+
RevisitReason.fromJSON = (json) => {
|
|
655
|
+
if (json == null) return json;
|
|
656
|
+
const tag = json["@@tagName"];
|
|
657
|
+
if (!tag) throw new TypeError(`RevisitReason.fromJSON: missing @@tagName on ${RuntimeForGeneratedTypes._toString(json)}`);
|
|
658
|
+
if (!RevisitReason["@@tagNames"].includes(tag))
|
|
659
|
+
throw new TypeError(`RevisitReason.fromJSON: unknown variant "${tag}"`);
|
|
660
|
+
return RevisitReason[tag]._from(json);
|
|
661
|
+
};
|
|
662
|
+
|
|
663
|
+
// src/corpus-db.js
|
|
664
|
+
var T = {
|
|
665
|
+
// Convert a Post Tagged type to a row object for SQLite insertion
|
|
666
|
+
// @sig toRow :: Post -> Object
|
|
667
|
+
toRow: ({ source, sourceId, channel, author, title, body, parentId, url, score, createdAt, postType }) => ({
|
|
668
|
+
source,
|
|
669
|
+
sourceId,
|
|
670
|
+
channel,
|
|
671
|
+
author,
|
|
672
|
+
title,
|
|
673
|
+
body,
|
|
674
|
+
parentId,
|
|
675
|
+
url,
|
|
676
|
+
score,
|
|
677
|
+
createdAt,
|
|
678
|
+
postType
|
|
679
|
+
}),
|
|
680
|
+
// Accumulate source counts from query rows into an object
|
|
681
|
+
// @sig toSourceCounts :: ([Object]) -> Object
|
|
682
|
+
toSourceCounts: (rows) => rows.reduce((acc, row) => ({ ...acc, [row.source]: row.count }), {}),
|
|
683
|
+
// Convert a RevisitReason TaggedSum to its database text representation
|
|
684
|
+
// @sig toReasonText :: RevisitReason -> String
|
|
685
|
+
toReasonText: (reason) => reason.match({ Recent: () => "recent", HighScore: () => "high_score", HasMore: () => "has_more" }),
|
|
686
|
+
// Convert a database reason text to a RevisitReason TaggedSum
|
|
687
|
+
// @sig toRevisitReason :: String -> RevisitReason
|
|
688
|
+
toRevisitReason: (text) => {
|
|
689
|
+
if (text === "recent") return RevisitReason.Recent();
|
|
690
|
+
if (text === "high_score") return RevisitReason.HighScore();
|
|
691
|
+
if (text === "has_more") return RevisitReason.HasMore();
|
|
692
|
+
throw new Error(`Unknown revisit reason: ${text}`);
|
|
693
|
+
},
|
|
694
|
+
// Convert a revisit_queue row to a domain object with RevisitReason TaggedSum
|
|
695
|
+
// @sig toQueueEntry :: Object -> Object
|
|
696
|
+
toQueueEntry: (row) => ({ ...row, reason: T.toRevisitReason(row.reason) })
|
|
697
|
+
};
|
|
698
|
+
var F = {
|
|
699
|
+
// Prepare the upsert statement — inserts new posts, updates score/scrapedAt on conflict
|
|
700
|
+
// @sig buildUpsertStmt :: Database -> Statement
|
|
701
|
+
buildUpsertStmt: (db) => db.prepare(upsertSql),
|
|
702
|
+
// Prepare the scrape run insert statement
|
|
703
|
+
// @sig buildScrapeRunStmt :: Database -> Statement
|
|
704
|
+
buildScrapeRunStmt: (db) => db.prepare(scrapeRunSql),
|
|
705
|
+
// Prepare the revisit queue upsert statement
|
|
706
|
+
// WHY: one entry per post — reason upgrades (most specific wins) but never downgrades
|
|
707
|
+
// @sig buildEnqueueStmt :: Database -> Statement
|
|
708
|
+
buildEnqueueStmt: (db) => db.prepare(enqueueSql)
|
|
709
|
+
};
|
|
710
|
+
var A = {
|
|
711
|
+
// Count truly new inserts within a transaction (upserts that update don't count)
|
|
712
|
+
// WHY: ON CONFLICT DO UPDATE reports changes=1 for both insert and update,
|
|
713
|
+
// so we compare total row count before/after to get actual insert count
|
|
714
|
+
// @sig countInserted :: (Database, Statement, [Object]) -> Number
|
|
715
|
+
countInserted: (db, stmt, rows) => {
|
|
716
|
+
const before = db.prepare("SELECT COUNT(*) as c FROM posts").get().c;
|
|
717
|
+
rows.forEach((row) => stmt.run(row));
|
|
718
|
+
const after = db.prepare("SELECT COUNT(*) as c FROM posts").get().c;
|
|
719
|
+
return after - before;
|
|
720
|
+
}
|
|
721
|
+
};
|
|
722
|
+
var __dirname = dirname(fileURLToPath(import.meta.url));
|
|
723
|
+
var schemaPath = resolve(__dirname, "../schema.sql");
|
|
724
|
+
var defaultDbPath = resolve(homedir(), ".community-intelligence", "corpus.db");
|
|
725
|
+
var upsertColumns = "source, sourceId, channel, author, title, body, parentId, url, score, createdAt, postType";
|
|
726
|
+
var upsertParams = "@source, @sourceId, @channel, @author, @title, @body, @parentId, @url, @score, @createdAt, @postType";
|
|
727
|
+
var upsertSql = `INSERT INTO posts (${upsertColumns}) VALUES (${upsertParams})
|
|
728
|
+
ON CONFLICT(source, sourceId) DO UPDATE SET score = excluded.score, scrapedAt = datetime('now')`;
|
|
729
|
+
var scrapeRunColumns = "source, channel, startedAt, completedAt, postsFound, postsInserted, commentsFound, commentsInserted";
|
|
730
|
+
var scrapeRunParams = "@source, @channel, @startedAt, @completedAt, @postsFound, @postsInserted, @commentsFound, @commentsInserted";
|
|
731
|
+
var scrapeRunSql = `INSERT INTO scrape_runs (${scrapeRunColumns}) VALUES (${scrapeRunParams})`;
|
|
732
|
+
var enqueueSql = `INSERT INTO revisit_queue (source, sourceId, reason, moreCount)
|
|
733
|
+
VALUES (@source, @sourceId, @reason, @moreCount)
|
|
734
|
+
ON CONFLICT(source, sourceId) DO UPDATE SET
|
|
735
|
+
reason = CASE
|
|
736
|
+
WHEN excluded.reason = 'has_more' THEN 'has_more'
|
|
737
|
+
WHEN excluded.reason = 'high_score' AND revisit_queue.reason != 'has_more' THEN 'high_score'
|
|
738
|
+
ELSE revisit_queue.reason
|
|
739
|
+
END,
|
|
740
|
+
moreCount = CASE
|
|
741
|
+
WHEN excluded.moreCount > revisit_queue.moreCount THEN excluded.moreCount
|
|
742
|
+
ELSE revisit_queue.moreCount
|
|
743
|
+
END`;
|
|
744
|
+
var topChannelsSql = "SELECT channel, source, COUNT(*) as count FROM posts GROUP BY channel, source ORDER BY count DESC LIMIT 10";
|
|
745
|
+
var createDb = (dbPath) => {
|
|
746
|
+
const resolvedPath = dbPath || defaultDbPath;
|
|
747
|
+
const dir = dirname(resolvedPath);
|
|
748
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
749
|
+
const db = Database(resolvedPath);
|
|
750
|
+
db.pragma("journal_mode = WAL");
|
|
751
|
+
db.exec(readFileSync(schemaPath, "utf-8"));
|
|
752
|
+
return db;
|
|
753
|
+
};
|
|
754
|
+
var persistPost = (db, post) => F.buildUpsertStmt(db).run(T.toRow(post));
|
|
755
|
+
var persistPosts = (db, posts) => {
|
|
756
|
+
const stmt = F.buildUpsertStmt(db);
|
|
757
|
+
const runAll = db.transaction((rows) => A.countInserted(db, stmt, rows));
|
|
758
|
+
return runAll(posts.map(T.toRow));
|
|
759
|
+
};
|
|
760
|
+
var persistScrapeRun = (db, run) => F.buildScrapeRunStmt(db).run(run);
|
|
761
|
+
var persistRevisitEntry = (db, { source, sourceId, reason, moreCount }) => F.buildEnqueueStmt(db).run({ source, sourceId, reason: T.toReasonText(reason), moreCount: moreCount || 0 });
|
|
762
|
+
var queryRevisitQueue = (db, source, channel) => {
|
|
763
|
+
const sql = channel ? `SELECT rq.* FROM revisit_queue rq JOIN posts p ON rq.source = p.source AND rq.sourceId = p.sourceId
|
|
764
|
+
WHERE rq.source = ? AND p.channel = ?` : "SELECT * FROM revisit_queue WHERE source = ?";
|
|
765
|
+
const params = channel ? [source, channel] : [source];
|
|
766
|
+
return db.prepare(sql).all(...params).map(T.toQueueEntry);
|
|
767
|
+
};
|
|
768
|
+
var updateQueueEntry = (db, { source, sourceId, unchangedCount, lastCheckedAt }) => db.prepare("UPDATE revisit_queue SET unchangedCount = ?, lastCheckedAt = ? WHERE source = ? AND sourceId = ?").run(unchangedCount, lastCheckedAt, source, sourceId);
|
|
769
|
+
var removeFromQueue = (db, source, sourceId) => db.prepare("DELETE FROM revisit_queue WHERE source = ? AND sourceId = ?").run(source, sourceId);
|
|
770
|
+
var queryPostBySourceId = (db, source, sourceId) => db.prepare("SELECT * FROM posts WHERE source = ? AND sourceId = ?").get(source, sourceId);
|
|
771
|
+
var countChildPosts = (db, parentId) => db.prepare("SELECT COUNT(*) as c FROM posts WHERE parentId = ?").get(parentId).c;
|
|
772
|
+
var queryScrapeRuns = (db, source, channel, limit) => {
|
|
773
|
+
const sql = channel ? "SELECT * FROM scrape_runs WHERE source = ? AND channel = ? ORDER BY completedAt DESC LIMIT ?" : "SELECT * FROM scrape_runs WHERE source = ? ORDER BY completedAt DESC LIMIT ?";
|
|
774
|
+
const params = channel ? [source, channel, limit || 10] : [source, limit || 10];
|
|
775
|
+
return db.prepare(sql).all(...params);
|
|
776
|
+
};
|
|
777
|
+
var queryLastCreatedAt = (db, source, channel) => {
|
|
778
|
+
const row = db.prepare("SELECT MAX(createdAt) as maxCreatedAt FROM posts WHERE source = ? AND channel = ?").get(source, channel);
|
|
779
|
+
return row?.maxCreatedAt ?? void 0;
|
|
780
|
+
};
|
|
781
|
+
var queryStatus = (db) => {
|
|
782
|
+
const totalRow = db.prepare("SELECT COUNT(*) as total FROM posts").get();
|
|
783
|
+
const dateRange = db.prepare("SELECT MIN(createdAt) as earliest, MAX(createdAt) as latest FROM posts").get();
|
|
784
|
+
const countRows = db.prepare("SELECT source, COUNT(*) as count FROM posts GROUP BY source").all();
|
|
785
|
+
const bySource = T.toSourceCounts(countRows);
|
|
786
|
+
const topChannels = db.prepare(topChannelsSql).all();
|
|
787
|
+
return { total: totalRow.total, earliest: dateRange.earliest, latest: dateRange.latest, bySource, topChannels };
|
|
788
|
+
};
|
|
789
|
+
var queryPosts = (db, { source, channel, text, author, since, until, minScore, limit } = {}) => {
|
|
790
|
+
const conditions = [];
|
|
791
|
+
const params = [];
|
|
792
|
+
if (source) {
|
|
793
|
+
conditions.push("source = ?");
|
|
794
|
+
params.push(source);
|
|
795
|
+
}
|
|
796
|
+
if (channel) {
|
|
797
|
+
conditions.push("channel = ?");
|
|
798
|
+
params.push(channel);
|
|
799
|
+
}
|
|
800
|
+
if (text) {
|
|
801
|
+
conditions.push("(body LIKE ? OR title LIKE ?)");
|
|
802
|
+
params.push(`%${text}%`, `%${text}%`);
|
|
803
|
+
}
|
|
804
|
+
if (author) {
|
|
805
|
+
conditions.push("author = ?");
|
|
806
|
+
params.push(author);
|
|
807
|
+
}
|
|
808
|
+
if (since) {
|
|
809
|
+
conditions.push("createdAt >= ?");
|
|
810
|
+
params.push(since);
|
|
811
|
+
}
|
|
812
|
+
if (until) {
|
|
813
|
+
conditions.push("createdAt <= ?");
|
|
814
|
+
params.push(until);
|
|
815
|
+
}
|
|
816
|
+
if (minScore !== void 0) {
|
|
817
|
+
conditions.push("score >= ?");
|
|
818
|
+
params.push(minScore);
|
|
819
|
+
}
|
|
820
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
821
|
+
const limitClause = limit ? "LIMIT ?" : "";
|
|
822
|
+
if (limit) params.push(Number(limit));
|
|
823
|
+
const sql = `SELECT * FROM posts ${where} ORDER BY createdAt DESC ${limitClause}`;
|
|
824
|
+
return db.prepare(sql).all(...params);
|
|
825
|
+
};
|
|
826
|
+
var CorpusDb = {
|
|
827
|
+
createDb,
|
|
828
|
+
persistPost,
|
|
829
|
+
persistPosts,
|
|
830
|
+
persistScrapeRun,
|
|
831
|
+
persistRevisitEntry,
|
|
832
|
+
queryLastCreatedAt,
|
|
833
|
+
queryStatus,
|
|
834
|
+
queryPosts,
|
|
835
|
+
queryPostBySourceId,
|
|
836
|
+
queryRevisitQueue,
|
|
837
|
+
queryScrapeRuns,
|
|
838
|
+
updateQueueEntry,
|
|
839
|
+
removeFromQueue,
|
|
840
|
+
countChildPosts
|
|
841
|
+
};
|
|
842
|
+
export {
|
|
843
|
+
CorpusDb
|
|
844
|
+
};
|