cursedbelt-core 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "Are these two files the same thing?" — the fleet's ONE answer, with no I/O in it.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this is here and not in an app ──────────────────────────────────────────────────
|
|
5
|
+
* `binary-server` wrote these rules first (2026-08-13, the owner: *"make strategy to always
|
|
6
|
+
* check for duplicate binaries in same app for same user … mark likely the duplicates for
|
|
7
|
+
* users to confirm"*), and they were measured against a real store: the copy-counter cap, the
|
|
8
|
+
* camera-generic stem list and the 5% size window each exist because a wider rule proposed
|
|
9
|
+
* thousands of pairs nobody could ever review. `music` then needed the same question asked of
|
|
10
|
+
* a 5,400-track library — *"I want to see a duplicate next to the file that it is a possible
|
|
11
|
+
* duplicate of … The 'binary-server' app (bs) already has this feature for checking duplicate
|
|
12
|
+
* files. We can reuse it and enhance the customizability if needed so the main strategy is
|
|
13
|
+
* shared."*
|
|
14
|
+
*
|
|
15
|
+
* So the strategy lives once, here, and both consume it. What each app keeps for itself is the
|
|
16
|
+
* evidence only it has: bs's perceptual frame hashes, music's artist/title key.
|
|
17
|
+
*
|
|
18
|
+
* ── What is NOT here, deliberately ──────────────────────────────────────────────────────
|
|
19
|
+
* Anything that needs a database, a filesystem, a fetch or a picture decoder. This module is
|
|
20
|
+
* pure so both consumers can prove their behaviour with `bun test` instead of against a live
|
|
21
|
+
* store, and so a browser bundle may import it (music renders the pair side by side).
|
|
22
|
+
*/
|
|
23
|
+
/**
|
|
24
|
+
* How far two sizes may differ and still be called the same thing.
|
|
25
|
+
*
|
|
26
|
+
* A re-encode at a different quality is the case this exists for, and those land within a few
|
|
27
|
+
* percent. Zero would make a name match a strict subset of "same name AND same size", which
|
|
28
|
+
* misses precisely the files worth asking about; much larger starts pairing a clip with its
|
|
29
|
+
* own trailer.
|
|
30
|
+
*/
|
|
31
|
+
export declare const SIZE_TOLERANCE = 0.05;
|
|
32
|
+
/**
|
|
33
|
+
* How far two playing times may differ and still be the same recording.
|
|
34
|
+
*
|
|
35
|
+
* Wide enough for a re-encode (a trim of a trailing black frame, a container that rounds), far
|
|
36
|
+
* too narrow for the owner's 2026-08-14 report of a 3:56 video paired with a 1:52 one.
|
|
37
|
+
*/
|
|
38
|
+
export declare const DURATION_TOLERANCE = 0.05;
|
|
39
|
+
/**
|
|
40
|
+
* The name to compare, reduced to what a human means by "the same file".
|
|
41
|
+
*
|
|
42
|
+
* Strips the directory, the extension, and the marks a copy picks up along the way —
|
|
43
|
+
* `Finder`'s ` copy`/` copy 2`, a browser's ` (1)`, a script's `_1`/`-1` — then collapses
|
|
44
|
+
* everything that is not a letter or a digit. `My Video (1).mp4`, `my_video.MP4` and
|
|
45
|
+
* `My-Video copy 2.mov` all reduce to `myvideo`.
|
|
46
|
+
*
|
|
47
|
+
* 🔴 The extension is stripped ON PURPOSE, so a re-encode (`.mov` → `.mp4`, `.wma` → `.mp3`)
|
|
48
|
+
* still pairs with its source. That is the single most common likely-duplicate in this fleet's
|
|
49
|
+
* stores and a rule that kept the extension would never see one.
|
|
50
|
+
*
|
|
51
|
+
* 🔴 **The copy-counter strip is capped at two digits**, and that cap is a measured bug fix
|
|
52
|
+
* (bs, 2026-08-14). It used to be `[-_]\d+$`, which eats a CAMERA SEQUENCE as readily as a copy
|
|
53
|
+
* marker: `IMG_4595` and `IMG_4646` both reduced to `img`, so every iPhone capture in a tenant
|
|
54
|
+
* shared one bucket and paired with every other one whose size was within 5%. `_1`…`_99` is
|
|
55
|
+
* what a copy marker actually looks like; four digits is a photo's identity, and it must
|
|
56
|
+
* survive.
|
|
57
|
+
*/
|
|
58
|
+
export declare function normalizeName(name: string | null | undefined): string;
|
|
59
|
+
/**
|
|
60
|
+
* Stems that name a CAMERA, not a file.
|
|
61
|
+
*
|
|
62
|
+
* `IMG`, `DSC`, `PXL`, `VID` are what a device calls everything it produces; two files whose
|
|
63
|
+
* names reduce to one of these have told us nothing except that they came from a phone. Even
|
|
64
|
+
* with the counter cap above, `IMG_1.MOV` still reduces to `img`, and a bucket called "img" is
|
|
65
|
+
* every photo the owner has ever taken.
|
|
66
|
+
*
|
|
67
|
+
* 🔴 This rejects a MATCH, never a file. Both copies stay wherever they are, and an
|
|
68
|
+
* identical-bytes pair is unaffected — a checksum match never consults the name.
|
|
69
|
+
*/
|
|
70
|
+
export declare const GENERIC_STEMS: ReadonlySet<string>;
|
|
71
|
+
/**
|
|
72
|
+
* A name that could belong to any file from that device tells us nothing about identity.
|
|
73
|
+
*
|
|
74
|
+
* 🔴 The trailing digit run is stripped BEFORE the lookup, and only here — never in
|
|
75
|
+
* {@link normalizeName}, where removing it is what made every camera capture one bucket. The
|
|
76
|
+
* two are asked at different moments: `normalizeName` decides whether two names are the SAME
|
|
77
|
+
* (so `img4595` and `img4646` must stay apart), and this decides whether a name that already
|
|
78
|
+
* matched is worth anything (so two files both called `Track01` have told us nothing, and a
|
|
79
|
+
* ripper names half a library that).
|
|
80
|
+
*/
|
|
81
|
+
export declare const isGenericName: (normalized: string) => boolean;
|
|
82
|
+
/** Within {@link SIZE_TOLERANCE} of each other. Two zero-byte files are not "close" — they
|
|
83
|
+
* are two files nothing is known about. */
|
|
84
|
+
export declare function sizesClose(a: number, b: number, tolerance?: number): boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Within {@link DURATION_TOLERANCE} of each other.
|
|
87
|
+
*
|
|
88
|
+
* 🔴 A missing or zero duration answers `null`, never `false`: "nobody has measured this yet"
|
|
89
|
+
* and "these are different lengths" are opposite answers, and collapsing them makes an
|
|
90
|
+
* unprobed library look like a library with no duplicates in it.
|
|
91
|
+
*/
|
|
92
|
+
export declare function durationsClose(a: number | null | undefined, b: number | null | undefined, tolerance?: number): boolean | null;
|
|
93
|
+
/** One file, reduced to what this module can reason about. Units are the caller's — pass
|
|
94
|
+
* both durations in the same unit or not at all. */
|
|
95
|
+
export interface NamedFileFacts {
|
|
96
|
+
/** The name a person would recognise: a display title, else the last path segment. */
|
|
97
|
+
name: string | null | undefined;
|
|
98
|
+
sizeBytes: number;
|
|
99
|
+
/** Playing time, in whatever unit the caller uses consistently. Null = never measured. */
|
|
100
|
+
duration?: number | null;
|
|
101
|
+
}
|
|
102
|
+
export interface NameMatchVerdict {
|
|
103
|
+
match: boolean;
|
|
104
|
+
/** Written for the owner reading a review queue, never for a log. Empty when `match` is
|
|
105
|
+
* false — a non-match has nothing to explain. */
|
|
106
|
+
reason: string;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* The shared "these look like the same thing" test: the NAME agrees after normalization, the
|
|
110
|
+
* sizes are close, and nothing about the length says otherwise.
|
|
111
|
+
*
|
|
112
|
+
* It is deliberately a QUESTION and never an action. Both consumers act only on a checksum,
|
|
113
|
+
* which is proof; this decides what is worth showing a person side by side.
|
|
114
|
+
*/
|
|
115
|
+
export declare function likelyTheSameFile(a: NamedFileFacts, b: NamedFileFacts): NameMatchVerdict;
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "Are these two files the same thing?" — the fleet's ONE answer, with no I/O in it.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this is here and not in an app ──────────────────────────────────────────────────
|
|
5
|
+
* `binary-server` wrote these rules first (2026-08-13, the owner: *"make strategy to always
|
|
6
|
+
* check for duplicate binaries in same app for same user … mark likely the duplicates for
|
|
7
|
+
* users to confirm"*), and they were measured against a real store: the copy-counter cap, the
|
|
8
|
+
* camera-generic stem list and the 5% size window each exist because a wider rule proposed
|
|
9
|
+
* thousands of pairs nobody could ever review. `music` then needed the same question asked of
|
|
10
|
+
* a 5,400-track library — *"I want to see a duplicate next to the file that it is a possible
|
|
11
|
+
* duplicate of … The 'binary-server' app (bs) already has this feature for checking duplicate
|
|
12
|
+
* files. We can reuse it and enhance the customizability if needed so the main strategy is
|
|
13
|
+
* shared."*
|
|
14
|
+
*
|
|
15
|
+
* So the strategy lives once, here, and both consume it. What each app keeps for itself is the
|
|
16
|
+
* evidence only it has: bs's perceptual frame hashes, music's artist/title key.
|
|
17
|
+
*
|
|
18
|
+
* ── What is NOT here, deliberately ──────────────────────────────────────────────────────
|
|
19
|
+
* Anything that needs a database, a filesystem, a fetch or a picture decoder. This module is
|
|
20
|
+
* pure so both consumers can prove their behaviour with `bun test` instead of against a live
|
|
21
|
+
* store, and so a browser bundle may import it (music renders the pair side by side).
|
|
22
|
+
*/
|
|
23
|
+
/**
|
|
24
|
+
* How far two sizes may differ and still be called the same thing.
|
|
25
|
+
*
|
|
26
|
+
* A re-encode at a different quality is the case this exists for, and those land within a few
|
|
27
|
+
* percent. Zero would make a name match a strict subset of "same name AND same size", which
|
|
28
|
+
* misses precisely the files worth asking about; much larger starts pairing a clip with its
|
|
29
|
+
* own trailer.
|
|
30
|
+
*/
|
|
31
|
+
export const SIZE_TOLERANCE = 0.05;
|
|
32
|
+
/**
|
|
33
|
+
* How far two playing times may differ and still be the same recording.
|
|
34
|
+
*
|
|
35
|
+
* Wide enough for a re-encode (a trim of a trailing black frame, a container that rounds), far
|
|
36
|
+
* too narrow for the owner's 2026-08-14 report of a 3:56 video paired with a 1:52 one.
|
|
37
|
+
*/
|
|
38
|
+
export const DURATION_TOLERANCE = 0.05;
|
|
39
|
+
/**
|
|
40
|
+
* The name to compare, reduced to what a human means by "the same file".
|
|
41
|
+
*
|
|
42
|
+
* Strips the directory, the extension, and the marks a copy picks up along the way —
|
|
43
|
+
* `Finder`'s ` copy`/` copy 2`, a browser's ` (1)`, a script's `_1`/`-1` — then collapses
|
|
44
|
+
* everything that is not a letter or a digit. `My Video (1).mp4`, `my_video.MP4` and
|
|
45
|
+
* `My-Video copy 2.mov` all reduce to `myvideo`.
|
|
46
|
+
*
|
|
47
|
+
* 🔴 The extension is stripped ON PURPOSE, so a re-encode (`.mov` → `.mp4`, `.wma` → `.mp3`)
|
|
48
|
+
* still pairs with its source. That is the single most common likely-duplicate in this fleet's
|
|
49
|
+
* stores and a rule that kept the extension would never see one.
|
|
50
|
+
*
|
|
51
|
+
* 🔴 **The copy-counter strip is capped at two digits**, and that cap is a measured bug fix
|
|
52
|
+
* (bs, 2026-08-14). It used to be `[-_]\d+$`, which eats a CAMERA SEQUENCE as readily as a copy
|
|
53
|
+
* marker: `IMG_4595` and `IMG_4646` both reduced to `img`, so every iPhone capture in a tenant
|
|
54
|
+
* shared one bucket and paired with every other one whose size was within 5%. `_1`…`_99` is
|
|
55
|
+
* what a copy marker actually looks like; four digits is a photo's identity, and it must
|
|
56
|
+
* survive.
|
|
57
|
+
*/
|
|
58
|
+
export function normalizeName(name) {
|
|
59
|
+
if (!name)
|
|
60
|
+
return "";
|
|
61
|
+
const leaf = name.split(/[\\/]/).pop() ?? "";
|
|
62
|
+
const stem = leaf.replace(/\.[a-z0-9]{1,5}$/i, "");
|
|
63
|
+
return stem
|
|
64
|
+
.toLowerCase()
|
|
65
|
+
.replace(/\s*\(\d+\)\s*$/, "")
|
|
66
|
+
.replace(/\s+copy(\s+\d+)?\s*$/, "")
|
|
67
|
+
.replace(/[-_]\d{1,2}\s*$/, "")
|
|
68
|
+
.replace(/[^a-z0-9]+/g, "");
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Stems that name a CAMERA, not a file.
|
|
72
|
+
*
|
|
73
|
+
* `IMG`, `DSC`, `PXL`, `VID` are what a device calls everything it produces; two files whose
|
|
74
|
+
* names reduce to one of these have told us nothing except that they came from a phone. Even
|
|
75
|
+
* with the counter cap above, `IMG_1.MOV` still reduces to `img`, and a bucket called "img" is
|
|
76
|
+
* every photo the owner has ever taken.
|
|
77
|
+
*
|
|
78
|
+
* 🔴 This rejects a MATCH, never a file. Both copies stay wherever they are, and an
|
|
79
|
+
* identical-bytes pair is unaffected — a checksum match never consults the name.
|
|
80
|
+
*/
|
|
81
|
+
export const GENERIC_STEMS = new Set([
|
|
82
|
+
"img",
|
|
83
|
+
"image",
|
|
84
|
+
"images",
|
|
85
|
+
"dsc",
|
|
86
|
+
"dscf",
|
|
87
|
+
"dscn",
|
|
88
|
+
"pxl",
|
|
89
|
+
"vid",
|
|
90
|
+
"video",
|
|
91
|
+
"movie",
|
|
92
|
+
"mov",
|
|
93
|
+
"photo",
|
|
94
|
+
"pic",
|
|
95
|
+
"picture",
|
|
96
|
+
"screenshot",
|
|
97
|
+
"screenrecording",
|
|
98
|
+
"untitled",
|
|
99
|
+
"unknown",
|
|
100
|
+
"file",
|
|
101
|
+
"download",
|
|
102
|
+
"scan",
|
|
103
|
+
"capture",
|
|
104
|
+
"copy",
|
|
105
|
+
"new",
|
|
106
|
+
"temp",
|
|
107
|
+
"tmp",
|
|
108
|
+
// Music's own version of the same problem: an untagged rip is called this by every ripper
|
|
109
|
+
// ever written, and `track01` from two albums is two different songs.
|
|
110
|
+
"track",
|
|
111
|
+
"audiotrack",
|
|
112
|
+
"unknownartist",
|
|
113
|
+
]);
|
|
114
|
+
/**
|
|
115
|
+
* A name that could belong to any file from that device tells us nothing about identity.
|
|
116
|
+
*
|
|
117
|
+
* 🔴 The trailing digit run is stripped BEFORE the lookup, and only here — never in
|
|
118
|
+
* {@link normalizeName}, where removing it is what made every camera capture one bucket. The
|
|
119
|
+
* two are asked at different moments: `normalizeName` decides whether two names are the SAME
|
|
120
|
+
* (so `img4595` and `img4646` must stay apart), and this decides whether a name that already
|
|
121
|
+
* matched is worth anything (so two files both called `Track01` have told us nothing, and a
|
|
122
|
+
* ripper names half a library that).
|
|
123
|
+
*/
|
|
124
|
+
export const isGenericName = (normalized) => normalized.length < 3 ||
|
|
125
|
+
GENERIC_STEMS.has(normalized) ||
|
|
126
|
+
GENERIC_STEMS.has(normalized.replace(/\d+$/, ""));
|
|
127
|
+
/** Within {@link SIZE_TOLERANCE} of each other. Two zero-byte files are not "close" — they
|
|
128
|
+
* are two files nothing is known about. */
|
|
129
|
+
export function sizesClose(a, b, tolerance = SIZE_TOLERANCE) {
|
|
130
|
+
if (a === b && a > 0)
|
|
131
|
+
return true;
|
|
132
|
+
const larger = Math.max(a, b);
|
|
133
|
+
if (larger <= 0)
|
|
134
|
+
return false;
|
|
135
|
+
return Math.abs(a - b) / larger <= tolerance;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Within {@link DURATION_TOLERANCE} of each other.
|
|
139
|
+
*
|
|
140
|
+
* 🔴 A missing or zero duration answers `null`, never `false`: "nobody has measured this yet"
|
|
141
|
+
* and "these are different lengths" are opposite answers, and collapsing them makes an
|
|
142
|
+
* unprobed library look like a library with no duplicates in it.
|
|
143
|
+
*/
|
|
144
|
+
export function durationsClose(a, b, tolerance = DURATION_TOLERANCE) {
|
|
145
|
+
if (!a || !b || a <= 0 || b <= 0)
|
|
146
|
+
return null;
|
|
147
|
+
return Math.abs(a - b) / Math.max(a, b) <= tolerance;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The shared "these look like the same thing" test: the NAME agrees after normalization, the
|
|
151
|
+
* sizes are close, and nothing about the length says otherwise.
|
|
152
|
+
*
|
|
153
|
+
* It is deliberately a QUESTION and never an action. Both consumers act only on a checksum,
|
|
154
|
+
* which is proof; this decides what is worth showing a person side by side.
|
|
155
|
+
*/
|
|
156
|
+
export function likelyTheSameFile(a, b) {
|
|
157
|
+
const na = normalizeName(a.name);
|
|
158
|
+
const nb = normalizeName(b.name);
|
|
159
|
+
// Two empty names are not a match with each other — it means neither file has a name worth
|
|
160
|
+
// comparing (a bare digest key, an untitled upload), and pairing those proposes every
|
|
161
|
+
// unnamed object as a duplicate of every other.
|
|
162
|
+
if (!na || na !== nb)
|
|
163
|
+
return { match: false, reason: "" };
|
|
164
|
+
if (isGenericName(na))
|
|
165
|
+
return { match: false, reason: "" };
|
|
166
|
+
if (!sizesClose(a.sizeBytes, b.sizeBytes))
|
|
167
|
+
return { match: false, reason: "" };
|
|
168
|
+
if (durationsClose(a.duration, b.duration) === false)
|
|
169
|
+
return { match: false, reason: "" };
|
|
170
|
+
const pct = Math.round((Math.abs(a.sizeBytes - b.sizeBytes) / Math.max(a.sizeBytes, b.sizeBytes, 1)) * 100);
|
|
171
|
+
return {
|
|
172
|
+
match: true,
|
|
173
|
+
reason: a.sizeBytes === b.sizeBytes
|
|
174
|
+
? "same name, same size, different bytes"
|
|
175
|
+
: `same name, size within ${pct}%, different bytes`,
|
|
176
|
+
};
|
|
177
|
+
}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-core",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
|
-
"description": "Framework-agnostic core of the cursedbelt split
|
|
6
|
+
"description": "Framework-agnostic core of the cursedbelt split — isomorphic domain logic. cwip below it, cursedbelt-server and cursedbelt (react) above it.",
|
|
7
7
|
"cursed": {
|
|
8
8
|
"schemaVersion": 1,
|
|
9
9
|
"app": {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"typecheck": "tsc -p tsconfig.build.json --noEmit",
|
|
21
21
|
"test": "NODE_ENV=development bun test src",
|
|
22
22
|
"build": "bun run clean && tsc -p tsconfig.build.json",
|
|
23
|
-
"paths": "bun
|
|
23
|
+
"paths": "bun run scripts/paths.ts",
|
|
24
24
|
"verify": "bun run paths && bun run typecheck && bun run build && bun run test",
|
|
25
25
|
"prepublishOnly": "bun run verify"
|
|
26
26
|
},
|
|
@@ -125,6 +125,12 @@
|
|
|
125
125
|
"source": "./src/core/download.ts",
|
|
126
126
|
"import": "./dist/core/download.js"
|
|
127
127
|
},
|
|
128
|
+
"./duplicates": {
|
|
129
|
+
"types": "./dist/core/duplicates/index.d.ts",
|
|
130
|
+
"bun": "./src/core/duplicates/index.ts",
|
|
131
|
+
"source": "./src/core/duplicates/index.ts",
|
|
132
|
+
"import": "./dist/core/duplicates/index.js"
|
|
133
|
+
},
|
|
128
134
|
"./errors": {
|
|
129
135
|
"types": "./dist/shared/errors.d.ts",
|
|
130
136
|
"bun": "./src/shared/errors.ts",
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shared duplicate rules, proved against the cases that produced them.
|
|
3
|
+
*
|
|
4
|
+
* Every assertion here is a measured incident from `binary-server` or `music`, not an
|
|
5
|
+
* invented example — see the module header for which.
|
|
6
|
+
*/
|
|
7
|
+
import { describe, expect, it } from "bun:test";
|
|
8
|
+
import {
|
|
9
|
+
DURATION_TOLERANCE,
|
|
10
|
+
durationsClose,
|
|
11
|
+
isGenericName,
|
|
12
|
+
likelyTheSameFile,
|
|
13
|
+
normalizeName,
|
|
14
|
+
SIZE_TOLERANCE,
|
|
15
|
+
sizesClose,
|
|
16
|
+
} from "./index";
|
|
17
|
+
|
|
18
|
+
describe("normalizeName", () => {
|
|
19
|
+
it("reduces the marks a copy picks up to the same stem", () => {
|
|
20
|
+
expect(normalizeName("My Video (1).mp4")).toBe("myvideo");
|
|
21
|
+
expect(normalizeName("my_video.MP4")).toBe("myvideo");
|
|
22
|
+
expect(normalizeName("My-Video copy 2.mov")).toBe("myvideo");
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("strips the extension, so a re-encode still pairs with its source", () => {
|
|
26
|
+
expect(normalizeName("Song.wma")).toBe(normalizeName("Song.mp3"));
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it("🔴 keeps a four-digit camera sequence — the 2026-08-14 flood", () => {
|
|
30
|
+
// `IMG_4595` and `IMG_4646` must NOT collapse onto each other.
|
|
31
|
+
expect(normalizeName("IMG_4595.MOV")).not.toBe(normalizeName("IMG_4646.MOV"));
|
|
32
|
+
// …while a two-digit copy marker still does.
|
|
33
|
+
expect(normalizeName("mix_2.mp3")).toBe(normalizeName("mix.mp3"));
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it("drops the directory", () => {
|
|
37
|
+
expect(normalizeName("Main 1/rap/Song.mp3")).toBe("song");
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
describe("isGenericName", () => {
|
|
42
|
+
it("rejects the stems that name a device rather than a file", () => {
|
|
43
|
+
expect(isGenericName(normalizeName("IMG_1.MOV"))).toBe(true);
|
|
44
|
+
expect(isGenericName(normalizeName("Track01.mp3"))).toBe(true);
|
|
45
|
+
expect(isGenericName(normalizeName("Between U.mp3"))).toBe(false);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
describe("sizesClose", () => {
|
|
50
|
+
it("accepts a re-encode and refuses a different file", () => {
|
|
51
|
+
expect(sizesClose(10_000_000, 10_400_000)).toBe(true);
|
|
52
|
+
expect(sizesClose(10_000_000, 20_000_000)).toBe(false);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("🔴 two zero-byte files are not close — nothing is known about either", () => {
|
|
56
|
+
expect(sizesClose(0, 0)).toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("is bounded by the exported tolerance", () => {
|
|
60
|
+
expect(sizesClose(100, 100 * (1 + SIZE_TOLERANCE))).toBe(true);
|
|
61
|
+
expect(sizesClose(100, 100 * (1 + SIZE_TOLERANCE * 2))).toBe(false);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
describe("durationsClose", () => {
|
|
66
|
+
it("🔴 answers null when nobody has measured, never false", () => {
|
|
67
|
+
expect(durationsClose(null, 200)).toBeNull();
|
|
68
|
+
expect(durationsClose(0, 200)).toBeNull();
|
|
69
|
+
expect(durationsClose(undefined, undefined)).toBeNull();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("refuses the owner's 3:56-vs-1:52 pair", () => {
|
|
73
|
+
expect(durationsClose(236, 112)).toBe(false);
|
|
74
|
+
expect(durationsClose(236, 236 * (1 + DURATION_TOLERANCE / 2))).toBe(true);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
describe("likelyTheSameFile", () => {
|
|
79
|
+
const facts = (name: string, sizeBytes: number, duration?: number | null) => ({
|
|
80
|
+
name,
|
|
81
|
+
sizeBytes,
|
|
82
|
+
...(duration === undefined ? {} : { duration }),
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("pairs a re-encode of the same song", () => {
|
|
86
|
+
const verdict = likelyTheSameFile(
|
|
87
|
+
facts("Between U.mp3", 8_000_000, 312),
|
|
88
|
+
facts("Between U.wma", 7_800_000, 312),
|
|
89
|
+
);
|
|
90
|
+
expect(verdict.match).toBe(true);
|
|
91
|
+
expect(verdict.reason).toContain("same name");
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("does not pair two different songs of a similar size", () => {
|
|
95
|
+
expect(likelyTheSameFile(facts("A.mp3", 8_000_000), facts("B.mp3", 8_010_000)).match).toBe(
|
|
96
|
+
false,
|
|
97
|
+
);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("🔴 a length that disagrees kills a name+size match", () => {
|
|
101
|
+
expect(
|
|
102
|
+
likelyTheSameFile(facts("Set.mp3", 8_000_000, 236), facts("Set.mp3", 8_100_000, 112)).match,
|
|
103
|
+
).toBe(false);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("🔴 an unmeasured length does not — that would find nothing in an unprobed library", () => {
|
|
107
|
+
expect(
|
|
108
|
+
likelyTheSameFile(facts("Set.mp3", 8_000_000, null), facts("Set.mp3", 8_100_000, null)).match,
|
|
109
|
+
).toBe(true);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("refuses a generic stem, and refuses two nameless files", () => {
|
|
113
|
+
expect(likelyTheSameFile(facts("IMG_1.MOV", 500), facts("IMG_1.MOV", 500)).match).toBe(false);
|
|
114
|
+
expect(likelyTheSameFile(facts("", 500), facts("", 500)).match).toBe(false);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "Are these two files the same thing?" — the fleet's ONE answer, with no I/O in it.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this is here and not in an app ──────────────────────────────────────────────────
|
|
5
|
+
* `binary-server` wrote these rules first (2026-08-13, the owner: *"make strategy to always
|
|
6
|
+
* check for duplicate binaries in same app for same user … mark likely the duplicates for
|
|
7
|
+
* users to confirm"*), and they were measured against a real store: the copy-counter cap, the
|
|
8
|
+
* camera-generic stem list and the 5% size window each exist because a wider rule proposed
|
|
9
|
+
* thousands of pairs nobody could ever review. `music` then needed the same question asked of
|
|
10
|
+
* a 5,400-track library — *"I want to see a duplicate next to the file that it is a possible
|
|
11
|
+
* duplicate of … The 'binary-server' app (bs) already has this feature for checking duplicate
|
|
12
|
+
* files. We can reuse it and enhance the customizability if needed so the main strategy is
|
|
13
|
+
* shared."*
|
|
14
|
+
*
|
|
15
|
+
* So the strategy lives once, here, and both consume it. What each app keeps for itself is the
|
|
16
|
+
* evidence only it has: bs's perceptual frame hashes, music's artist/title key.
|
|
17
|
+
*
|
|
18
|
+
* ── What is NOT here, deliberately ──────────────────────────────────────────────────────
|
|
19
|
+
* Anything that needs a database, a filesystem, a fetch or a picture decoder. This module is
|
|
20
|
+
* pure so both consumers can prove their behaviour with `bun test` instead of against a live
|
|
21
|
+
* store, and so a browser bundle may import it (music renders the pair side by side).
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* How far two sizes may differ and still be called the same thing.
|
|
26
|
+
*
|
|
27
|
+
* A re-encode at a different quality is the case this exists for, and those land within a few
|
|
28
|
+
* percent. Zero would make a name match a strict subset of "same name AND same size", which
|
|
29
|
+
* misses precisely the files worth asking about; much larger starts pairing a clip with its
|
|
30
|
+
* own trailer.
|
|
31
|
+
*/
|
|
32
|
+
export const SIZE_TOLERANCE = 0.05;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* How far two playing times may differ and still be the same recording.
|
|
36
|
+
*
|
|
37
|
+
* Wide enough for a re-encode (a trim of a trailing black frame, a container that rounds), far
|
|
38
|
+
* too narrow for the owner's 2026-08-14 report of a 3:56 video paired with a 1:52 one.
|
|
39
|
+
*/
|
|
40
|
+
export const DURATION_TOLERANCE = 0.05;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The name to compare, reduced to what a human means by "the same file".
|
|
44
|
+
*
|
|
45
|
+
* Strips the directory, the extension, and the marks a copy picks up along the way —
|
|
46
|
+
* `Finder`'s ` copy`/` copy 2`, a browser's ` (1)`, a script's `_1`/`-1` — then collapses
|
|
47
|
+
* everything that is not a letter or a digit. `My Video (1).mp4`, `my_video.MP4` and
|
|
48
|
+
* `My-Video copy 2.mov` all reduce to `myvideo`.
|
|
49
|
+
*
|
|
50
|
+
* 🔴 The extension is stripped ON PURPOSE, so a re-encode (`.mov` → `.mp4`, `.wma` → `.mp3`)
|
|
51
|
+
* still pairs with its source. That is the single most common likely-duplicate in this fleet's
|
|
52
|
+
* stores and a rule that kept the extension would never see one.
|
|
53
|
+
*
|
|
54
|
+
* 🔴 **The copy-counter strip is capped at two digits**, and that cap is a measured bug fix
|
|
55
|
+
* (bs, 2026-08-14). It used to be `[-_]\d+$`, which eats a CAMERA SEQUENCE as readily as a copy
|
|
56
|
+
* marker: `IMG_4595` and `IMG_4646` both reduced to `img`, so every iPhone capture in a tenant
|
|
57
|
+
* shared one bucket and paired with every other one whose size was within 5%. `_1`…`_99` is
|
|
58
|
+
* what a copy marker actually looks like; four digits is a photo's identity, and it must
|
|
59
|
+
* survive.
|
|
60
|
+
*/
|
|
61
|
+
export function normalizeName(name: string | null | undefined): string {
|
|
62
|
+
if (!name) return "";
|
|
63
|
+
const leaf = name.split(/[\\/]/).pop() ?? "";
|
|
64
|
+
const stem = leaf.replace(/\.[a-z0-9]{1,5}$/i, "");
|
|
65
|
+
return stem
|
|
66
|
+
.toLowerCase()
|
|
67
|
+
.replace(/\s*\(\d+\)\s*$/, "")
|
|
68
|
+
.replace(/\s+copy(\s+\d+)?\s*$/, "")
|
|
69
|
+
.replace(/[-_]\d{1,2}\s*$/, "")
|
|
70
|
+
.replace(/[^a-z0-9]+/g, "");
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Stems that name a CAMERA, not a file.
|
|
75
|
+
*
|
|
76
|
+
* `IMG`, `DSC`, `PXL`, `VID` are what a device calls everything it produces; two files whose
|
|
77
|
+
* names reduce to one of these have told us nothing except that they came from a phone. Even
|
|
78
|
+
* with the counter cap above, `IMG_1.MOV` still reduces to `img`, and a bucket called "img" is
|
|
79
|
+
* every photo the owner has ever taken.
|
|
80
|
+
*
|
|
81
|
+
* 🔴 This rejects a MATCH, never a file. Both copies stay wherever they are, and an
|
|
82
|
+
* identical-bytes pair is unaffected — a checksum match never consults the name.
|
|
83
|
+
*/
|
|
84
|
+
export const GENERIC_STEMS: ReadonlySet<string> = new Set([
|
|
85
|
+
"img",
|
|
86
|
+
"image",
|
|
87
|
+
"images",
|
|
88
|
+
"dsc",
|
|
89
|
+
"dscf",
|
|
90
|
+
"dscn",
|
|
91
|
+
"pxl",
|
|
92
|
+
"vid",
|
|
93
|
+
"video",
|
|
94
|
+
"movie",
|
|
95
|
+
"mov",
|
|
96
|
+
"photo",
|
|
97
|
+
"pic",
|
|
98
|
+
"picture",
|
|
99
|
+
"screenshot",
|
|
100
|
+
"screenrecording",
|
|
101
|
+
"untitled",
|
|
102
|
+
"unknown",
|
|
103
|
+
"file",
|
|
104
|
+
"download",
|
|
105
|
+
"scan",
|
|
106
|
+
"capture",
|
|
107
|
+
"copy",
|
|
108
|
+
"new",
|
|
109
|
+
"temp",
|
|
110
|
+
"tmp",
|
|
111
|
+
// Music's own version of the same problem: an untagged rip is called this by every ripper
|
|
112
|
+
// ever written, and `track01` from two albums is two different songs.
|
|
113
|
+
"track",
|
|
114
|
+
"audiotrack",
|
|
115
|
+
"unknownartist",
|
|
116
|
+
]);
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* A name that could belong to any file from that device tells us nothing about identity.
|
|
120
|
+
*
|
|
121
|
+
* 🔴 The trailing digit run is stripped BEFORE the lookup, and only here — never in
|
|
122
|
+
* {@link normalizeName}, where removing it is what made every camera capture one bucket. The
|
|
123
|
+
* two are asked at different moments: `normalizeName` decides whether two names are the SAME
|
|
124
|
+
* (so `img4595` and `img4646` must stay apart), and this decides whether a name that already
|
|
125
|
+
* matched is worth anything (so two files both called `Track01` have told us nothing, and a
|
|
126
|
+
* ripper names half a library that).
|
|
127
|
+
*/
|
|
128
|
+
export const isGenericName = (normalized: string): boolean =>
|
|
129
|
+
normalized.length < 3 ||
|
|
130
|
+
GENERIC_STEMS.has(normalized) ||
|
|
131
|
+
GENERIC_STEMS.has(normalized.replace(/\d+$/, ""));
|
|
132
|
+
|
|
133
|
+
/** Within {@link SIZE_TOLERANCE} of each other. Two zero-byte files are not "close" — they
|
|
134
|
+
* are two files nothing is known about. */
|
|
135
|
+
export function sizesClose(a: number, b: number, tolerance = SIZE_TOLERANCE): boolean {
|
|
136
|
+
if (a === b && a > 0) return true;
|
|
137
|
+
const larger = Math.max(a, b);
|
|
138
|
+
if (larger <= 0) return false;
|
|
139
|
+
return Math.abs(a - b) / larger <= tolerance;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Within {@link DURATION_TOLERANCE} of each other.
|
|
144
|
+
*
|
|
145
|
+
* 🔴 A missing or zero duration answers `null`, never `false`: "nobody has measured this yet"
|
|
146
|
+
* and "these are different lengths" are opposite answers, and collapsing them makes an
|
|
147
|
+
* unprobed library look like a library with no duplicates in it.
|
|
148
|
+
*/
|
|
149
|
+
export function durationsClose(
|
|
150
|
+
a: number | null | undefined,
|
|
151
|
+
b: number | null | undefined,
|
|
152
|
+
tolerance = DURATION_TOLERANCE,
|
|
153
|
+
): boolean | null {
|
|
154
|
+
if (!a || !b || a <= 0 || b <= 0) return null;
|
|
155
|
+
return Math.abs(a - b) / Math.max(a, b) <= tolerance;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** One file, reduced to what this module can reason about. Units are the caller's — pass
|
|
159
|
+
* both durations in the same unit or not at all. */
|
|
160
|
+
export interface NamedFileFacts {
|
|
161
|
+
/** The name a person would recognise: a display title, else the last path segment. */
|
|
162
|
+
name: string | null | undefined;
|
|
163
|
+
sizeBytes: number;
|
|
164
|
+
/** Playing time, in whatever unit the caller uses consistently. Null = never measured. */
|
|
165
|
+
duration?: number | null;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export interface NameMatchVerdict {
|
|
169
|
+
match: boolean;
|
|
170
|
+
/** Written for the owner reading a review queue, never for a log. Empty when `match` is
|
|
171
|
+
* false — a non-match has nothing to explain. */
|
|
172
|
+
reason: string;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* The shared "these look like the same thing" test: the NAME agrees after normalization, the
|
|
177
|
+
* sizes are close, and nothing about the length says otherwise.
|
|
178
|
+
*
|
|
179
|
+
* It is deliberately a QUESTION and never an action. Both consumers act only on a checksum,
|
|
180
|
+
* which is proof; this decides what is worth showing a person side by side.
|
|
181
|
+
*/
|
|
182
|
+
export function likelyTheSameFile(a: NamedFileFacts, b: NamedFileFacts): NameMatchVerdict {
|
|
183
|
+
const na = normalizeName(a.name);
|
|
184
|
+
const nb = normalizeName(b.name);
|
|
185
|
+
// Two empty names are not a match with each other — it means neither file has a name worth
|
|
186
|
+
// comparing (a bare digest key, an untitled upload), and pairing those proposes every
|
|
187
|
+
// unnamed object as a duplicate of every other.
|
|
188
|
+
if (!na || na !== nb) return { match: false, reason: "" };
|
|
189
|
+
if (isGenericName(na)) return { match: false, reason: "" };
|
|
190
|
+
if (!sizesClose(a.sizeBytes, b.sizeBytes)) return { match: false, reason: "" };
|
|
191
|
+
if (durationsClose(a.duration, b.duration) === false) return { match: false, reason: "" };
|
|
192
|
+
|
|
193
|
+
const pct = Math.round(
|
|
194
|
+
(Math.abs(a.sizeBytes - b.sizeBytes) / Math.max(a.sizeBytes, b.sizeBytes, 1)) * 100,
|
|
195
|
+
);
|
|
196
|
+
return {
|
|
197
|
+
match: true,
|
|
198
|
+
reason:
|
|
199
|
+
a.sizeBytes === b.sizeBytes
|
|
200
|
+
? "same name, same size, different bytes"
|
|
201
|
+
: `same name, size within ${pct}%, different bytes`,
|
|
202
|
+
};
|
|
203
|
+
}
|