@linkprobe/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +155 -0
- package/dist/index.d.mts +92 -0
- package/dist/index.d.ts +92 -0
- package/dist/index.js +243 -0
- package/dist/index.mjs +212 -0
- package/package.json +40 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Hossein Hosseini
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# @linkprobe/core
|
|
2
|
+
|
|
3
|
+
The framework-agnostic metadata extraction engine for LinkProbe.
|
|
4
|
+
|
|
5
|
+
`@linkprobe/core` provides utilities for extracting and resolving webpage metadata from URLs. It supports Open Graph, Twitter Cards, and standard HTML metadata, making it ideal for building link previews, content cards, social embeds, and metadata-driven experiences.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pnpm add @linkprobe/core
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Quick Start
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import {probe} from "@linkprobe/core";
|
|
17
|
+
|
|
18
|
+
const result = await probe("https://github.com");
|
|
19
|
+
|
|
20
|
+
if (result.status === "success") {
|
|
21
|
+
console.log(result.metadata);
|
|
22
|
+
}
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Example Response
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
{
|
|
29
|
+
status: "success",
|
|
30
|
+
metadata: {
|
|
31
|
+
url: "https://github.com",
|
|
32
|
+
openGraph: {
|
|
33
|
+
title: "GitHub",
|
|
34
|
+
description: "...",
|
|
35
|
+
image: "..."
|
|
36
|
+
},
|
|
37
|
+
twitter: {
|
|
38
|
+
title: "GitHub"
|
|
39
|
+
},
|
|
40
|
+
html: {
|
|
41
|
+
title: "GitHub"
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
page: {
|
|
45
|
+
requestedUrl: "https://github.com",
|
|
46
|
+
finalUrl: "https://github.com",
|
|
47
|
+
contentType: "text/html"
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Metadata Resolution
|
|
53
|
+
|
|
54
|
+
Metadata may be available from multiple sources.
|
|
55
|
+
|
|
56
|
+
Use `resolveMetadata()` to create a unified representation using customizable priority rules.
|
|
57
|
+
|
|
58
|
+
```ts
|
|
59
|
+
import {probe, resolveMetadata} from "@linkprobe/core";
|
|
60
|
+
|
|
61
|
+
const result = await probe("https://github.com");
|
|
62
|
+
|
|
63
|
+
if (result.status === "success") {
|
|
64
|
+
const metadata = resolveMetadata(result.metadata);
|
|
65
|
+
|
|
66
|
+
console.log(metadata);
|
|
67
|
+
}
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Example output:
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
{
|
|
74
|
+
title: "GitHub",
|
|
75
|
+
description: "...",
|
|
76
|
+
image: "...",
|
|
77
|
+
favicon: "...",
|
|
78
|
+
siteName: "GitHub"
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Custom Priorities
|
|
83
|
+
|
|
84
|
+
```ts
|
|
85
|
+
const metadata = resolveMetadata(result.metadata, {
|
|
86
|
+
titlePriority: ["twitter", "openGraph", "html"],
|
|
87
|
+
});
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Options
|
|
91
|
+
|
|
92
|
+
### probe(url, options)
|
|
93
|
+
|
|
94
|
+
```ts
|
|
95
|
+
await probe("https://github.com", {
|
|
96
|
+
timeout: 5000,
|
|
97
|
+
});
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
| Option | Type | Description |
|
|
101
|
+
| ------- | ------ | ------------------------------- |
|
|
102
|
+
| timeout | number | Request timeout in milliseconds |
|
|
103
|
+
|
|
104
|
+
## Supported Sources
|
|
105
|
+
|
|
106
|
+
### Open Graph
|
|
107
|
+
|
|
108
|
+
```html
|
|
109
|
+
<meta property="og:title" />
|
|
110
|
+
<meta property="og:description" />
|
|
111
|
+
<meta property="og:image" />
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Twitter Cards
|
|
115
|
+
|
|
116
|
+
```html
|
|
117
|
+
<meta name="twitter:title" />
|
|
118
|
+
<meta name="twitter:description" />
|
|
119
|
+
<meta name="twitter:image" />
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### HTML Metadata
|
|
123
|
+
|
|
124
|
+
```html
|
|
125
|
+
<title>...</title>
|
|
126
|
+
<meta name="description" />
|
|
127
|
+
<link rel="icon" />
|
|
128
|
+
<link rel="canonical" />
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Error Handling
|
|
132
|
+
|
|
133
|
+
```ts
|
|
134
|
+
const result = await probe(url);
|
|
135
|
+
|
|
136
|
+
if (result.status === "failed") {
|
|
137
|
+
console.error(result.error.code, result.error.message);
|
|
138
|
+
}
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Possible error codes:
|
|
142
|
+
|
|
143
|
+
```txt
|
|
144
|
+
FETCH_FAILED
|
|
145
|
+
TIMEOUT
|
|
146
|
+
UNKNOWN_ERROR
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
## TypeScript
|
|
150
|
+
|
|
151
|
+
All public APIs are fully typed and exported by the package.
|
|
152
|
+
|
|
153
|
+
```ts
|
|
154
|
+
import type {Metadata, ProbeResult, ProbeOptions, ResolvedMetadata} from "@linkprobe/core";
|
|
155
|
+
```
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
interface OpenGraphMetadata {
|
|
2
|
+
title?: string;
|
|
3
|
+
description?: string;
|
|
4
|
+
image?: string;
|
|
5
|
+
siteName?: string;
|
|
6
|
+
type?: string;
|
|
7
|
+
locale?: string;
|
|
8
|
+
}
|
|
9
|
+
interface TwitterMetadata {
|
|
10
|
+
title?: string;
|
|
11
|
+
description?: string;
|
|
12
|
+
image?: string;
|
|
13
|
+
site?: string;
|
|
14
|
+
creator?: string;
|
|
15
|
+
}
|
|
16
|
+
interface HtmlMetadata {
|
|
17
|
+
title?: string;
|
|
18
|
+
description?: string;
|
|
19
|
+
favicon?: string;
|
|
20
|
+
canonicalUrl?: string;
|
|
21
|
+
}
|
|
22
|
+
interface Metadata {
|
|
23
|
+
url: string;
|
|
24
|
+
openGraph?: OpenGraphMetadata;
|
|
25
|
+
twitter?: TwitterMetadata;
|
|
26
|
+
html?: HtmlMetadata;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface ProbeOptions {
|
|
30
|
+
timeout?: number;
|
|
31
|
+
}
|
|
32
|
+
interface PageInfo {
|
|
33
|
+
requestedUrl: string;
|
|
34
|
+
finalUrl: string;
|
|
35
|
+
contentType?: string;
|
|
36
|
+
}
|
|
37
|
+
interface SourceDiagnostics {
|
|
38
|
+
detected: boolean;
|
|
39
|
+
fields: string[];
|
|
40
|
+
}
|
|
41
|
+
interface ProbeDiagnostics {
|
|
42
|
+
openGraph?: SourceDiagnostics;
|
|
43
|
+
twitter?: SourceDiagnostics;
|
|
44
|
+
html?: SourceDiagnostics;
|
|
45
|
+
}
|
|
46
|
+
interface ProbeSuccess {
|
|
47
|
+
status: "success";
|
|
48
|
+
metadata: Metadata;
|
|
49
|
+
page: PageInfo;
|
|
50
|
+
diagnostics?: ProbeDiagnostics;
|
|
51
|
+
}
|
|
52
|
+
interface ProbeError {
|
|
53
|
+
code: string;
|
|
54
|
+
message: string;
|
|
55
|
+
}
|
|
56
|
+
interface ProbeFailed {
|
|
57
|
+
status: "failed";
|
|
58
|
+
error: ProbeError;
|
|
59
|
+
}
|
|
60
|
+
type ProbeResult = ProbeSuccess | ProbeFailed;
|
|
61
|
+
|
|
62
|
+
type MetadataSource = "openGraph" | "twitter" | "html";
|
|
63
|
+
interface ResolveMetadataOptions {
|
|
64
|
+
titlePriority?: MetadataSource[];
|
|
65
|
+
descriptionPriority?: MetadataSource[];
|
|
66
|
+
imagePriority?: MetadataSource[];
|
|
67
|
+
}
|
|
68
|
+
interface ResolvedMetadata {
|
|
69
|
+
title?: string;
|
|
70
|
+
description?: string;
|
|
71
|
+
image?: string;
|
|
72
|
+
favicon?: string;
|
|
73
|
+
siteName?: string;
|
|
74
|
+
canonicalUrl?: string;
|
|
75
|
+
}
|
|
76
|
+
interface MetadataCoverage {
|
|
77
|
+
score: number;
|
|
78
|
+
available: string[];
|
|
79
|
+
missing: string[];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
declare function buildDiagnostics(metadata: Metadata): ProbeDiagnostics;
|
|
83
|
+
|
|
84
|
+
declare function getMetadataCoverage(metadata: Metadata): MetadataCoverage;
|
|
85
|
+
|
|
86
|
+
declare function probe(url: string, options?: ProbeOptions): Promise<ProbeResult>;
|
|
87
|
+
|
|
88
|
+
declare function probeMany(urls: string[], options?: ProbeOptions): Promise<ProbeResult[]>;
|
|
89
|
+
|
|
90
|
+
declare function resolveMetadata(metadata: Metadata, options?: ResolveMetadataOptions): ResolvedMetadata;
|
|
91
|
+
|
|
92
|
+
export { buildDiagnostics, getMetadataCoverage, probe, probeMany, resolveMetadata };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
interface OpenGraphMetadata {
|
|
2
|
+
title?: string;
|
|
3
|
+
description?: string;
|
|
4
|
+
image?: string;
|
|
5
|
+
siteName?: string;
|
|
6
|
+
type?: string;
|
|
7
|
+
locale?: string;
|
|
8
|
+
}
|
|
9
|
+
interface TwitterMetadata {
|
|
10
|
+
title?: string;
|
|
11
|
+
description?: string;
|
|
12
|
+
image?: string;
|
|
13
|
+
site?: string;
|
|
14
|
+
creator?: string;
|
|
15
|
+
}
|
|
16
|
+
interface HtmlMetadata {
|
|
17
|
+
title?: string;
|
|
18
|
+
description?: string;
|
|
19
|
+
favicon?: string;
|
|
20
|
+
canonicalUrl?: string;
|
|
21
|
+
}
|
|
22
|
+
interface Metadata {
|
|
23
|
+
url: string;
|
|
24
|
+
openGraph?: OpenGraphMetadata;
|
|
25
|
+
twitter?: TwitterMetadata;
|
|
26
|
+
html?: HtmlMetadata;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface ProbeOptions {
|
|
30
|
+
timeout?: number;
|
|
31
|
+
}
|
|
32
|
+
interface PageInfo {
|
|
33
|
+
requestedUrl: string;
|
|
34
|
+
finalUrl: string;
|
|
35
|
+
contentType?: string;
|
|
36
|
+
}
|
|
37
|
+
interface SourceDiagnostics {
|
|
38
|
+
detected: boolean;
|
|
39
|
+
fields: string[];
|
|
40
|
+
}
|
|
41
|
+
interface ProbeDiagnostics {
|
|
42
|
+
openGraph?: SourceDiagnostics;
|
|
43
|
+
twitter?: SourceDiagnostics;
|
|
44
|
+
html?: SourceDiagnostics;
|
|
45
|
+
}
|
|
46
|
+
interface ProbeSuccess {
|
|
47
|
+
status: "success";
|
|
48
|
+
metadata: Metadata;
|
|
49
|
+
page: PageInfo;
|
|
50
|
+
diagnostics?: ProbeDiagnostics;
|
|
51
|
+
}
|
|
52
|
+
interface ProbeError {
|
|
53
|
+
code: string;
|
|
54
|
+
message: string;
|
|
55
|
+
}
|
|
56
|
+
interface ProbeFailed {
|
|
57
|
+
status: "failed";
|
|
58
|
+
error: ProbeError;
|
|
59
|
+
}
|
|
60
|
+
type ProbeResult = ProbeSuccess | ProbeFailed;
|
|
61
|
+
|
|
62
|
+
type MetadataSource = "openGraph" | "twitter" | "html";
|
|
63
|
+
interface ResolveMetadataOptions {
|
|
64
|
+
titlePriority?: MetadataSource[];
|
|
65
|
+
descriptionPriority?: MetadataSource[];
|
|
66
|
+
imagePriority?: MetadataSource[];
|
|
67
|
+
}
|
|
68
|
+
interface ResolvedMetadata {
|
|
69
|
+
title?: string;
|
|
70
|
+
description?: string;
|
|
71
|
+
image?: string;
|
|
72
|
+
favicon?: string;
|
|
73
|
+
siteName?: string;
|
|
74
|
+
canonicalUrl?: string;
|
|
75
|
+
}
|
|
76
|
+
interface MetadataCoverage {
|
|
77
|
+
score: number;
|
|
78
|
+
available: string[];
|
|
79
|
+
missing: string[];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
declare function buildDiagnostics(metadata: Metadata): ProbeDiagnostics;
|
|
83
|
+
|
|
84
|
+
declare function getMetadataCoverage(metadata: Metadata): MetadataCoverage;
|
|
85
|
+
|
|
86
|
+
declare function probe(url: string, options?: ProbeOptions): Promise<ProbeResult>;
|
|
87
|
+
|
|
88
|
+
declare function probeMany(urls: string[], options?: ProbeOptions): Promise<ProbeResult[]>;
|
|
89
|
+
|
|
90
|
+
declare function resolveMetadata(metadata: Metadata, options?: ResolveMetadataOptions): ResolvedMetadata;
|
|
91
|
+
|
|
92
|
+
export { buildDiagnostics, getMetadataCoverage, probe, probeMany, resolveMetadata };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/index.ts
|
|
21
|
+
var index_exports = {};
|
|
22
|
+
__export(index_exports, {
|
|
23
|
+
buildDiagnostics: () => buildDiagnostics,
|
|
24
|
+
getMetadataCoverage: () => getMetadataCoverage,
|
|
25
|
+
probe: () => probe,
|
|
26
|
+
probeMany: () => probeMany,
|
|
27
|
+
resolveMetadata: () => resolveMetadata
|
|
28
|
+
});
|
|
29
|
+
module.exports = __toCommonJS(index_exports);
|
|
30
|
+
|
|
31
|
+
// src/utils/extract-fields.ts
|
|
32
|
+
function extractFields(source) {
|
|
33
|
+
if (!source) {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
return Object.entries(source).filter(([, value]) => value !== void 0 && value !== null && value !== "").map(([key]) => key);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// src/build-diagnostics.ts
|
|
40
|
+
function buildDiagnostics(metadata) {
|
|
41
|
+
const openGraphFields = extractFields(metadata.openGraph);
|
|
42
|
+
const twitterFields = extractFields(metadata.twitter);
|
|
43
|
+
const htmlFields = extractFields(metadata.html);
|
|
44
|
+
return {
|
|
45
|
+
html: {
|
|
46
|
+
detected: htmlFields.length > 0,
|
|
47
|
+
fields: htmlFields
|
|
48
|
+
},
|
|
49
|
+
openGraph: {
|
|
50
|
+
detected: openGraphFields.length > 0,
|
|
51
|
+
fields: openGraphFields
|
|
52
|
+
},
|
|
53
|
+
twitter: {
|
|
54
|
+
detected: twitterFields.length > 0,
|
|
55
|
+
fields: twitterFields
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// src/resolve-metadata.ts
|
|
61
|
+
var DEFAULT_PRIORITY = ["openGraph", "twitter", "html"];
|
|
62
|
+
function resolveField(metadata, field, priority) {
|
|
63
|
+
for (const source of priority) {
|
|
64
|
+
const value = metadata[source]?.[field];
|
|
65
|
+
if (value !== void 0 && value !== null && value !== "") {
|
|
66
|
+
return value;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return void 0;
|
|
70
|
+
}
|
|
71
|
+
function resolveMetadata(metadata, options) {
|
|
72
|
+
return {
|
|
73
|
+
canonicalUrl: metadata.html?.canonicalUrl,
|
|
74
|
+
description: resolveField(
|
|
75
|
+
metadata,
|
|
76
|
+
"description",
|
|
77
|
+
options?.descriptionPriority ?? DEFAULT_PRIORITY
|
|
78
|
+
),
|
|
79
|
+
favicon: metadata.html?.favicon,
|
|
80
|
+
image: resolveField(metadata, "image", options?.imagePriority ?? DEFAULT_PRIORITY),
|
|
81
|
+
siteName: metadata.openGraph?.siteName,
|
|
82
|
+
title: resolveField(metadata, "title", options?.titlePriority ?? DEFAULT_PRIORITY)
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// src/get-metadata-coverage.ts
|
|
87
|
+
var REQUIRED_FIELDS = ["title", "description", "image", "favicon"];
|
|
88
|
+
function getMetadataCoverage(metadata) {
|
|
89
|
+
const resolved = resolveMetadata(metadata);
|
|
90
|
+
const available = REQUIRED_FIELDS.filter((field) => resolved[field]);
|
|
91
|
+
const missing = REQUIRED_FIELDS.filter((field) => !resolved[field]);
|
|
92
|
+
return {
|
|
93
|
+
available: [...available],
|
|
94
|
+
missing: [...missing],
|
|
95
|
+
score: Math.round(available.length / REQUIRED_FIELDS.length * 100)
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// src/probe.ts
|
|
100
|
+
var import_cheerio = require("cheerio");
|
|
101
|
+
|
|
102
|
+
// src/utils/resolve-url.ts
|
|
103
|
+
function resolveUrl(url, baseUrl) {
|
|
104
|
+
try {
|
|
105
|
+
return new URL(url, baseUrl).toString();
|
|
106
|
+
} catch {
|
|
107
|
+
return url;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// src/parsers/html-meta-parser.ts
|
|
112
|
+
var HtmlMetaParser = class {
|
|
113
|
+
parse(context) {
|
|
114
|
+
const { $, pageUrl } = context;
|
|
115
|
+
return {
|
|
116
|
+
canonicalUrl: $('link[rel="canonical"]').attr("href"),
|
|
117
|
+
description: $('meta[name="description"]').attr("content"),
|
|
118
|
+
favicon: resolveUrl(
|
|
119
|
+
$('link[rel="icon"]').attr("href") ?? $('link[rel="shortcut icon"]').attr("href") ?? $('link[rel="apple-touch-icon"]').attr("href") ?? "/favicon.ico",
|
|
120
|
+
pageUrl
|
|
121
|
+
),
|
|
122
|
+
title: $("title").text()
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
// src/parsers/open-graph-parser.ts
|
|
128
|
+
var OpenGraphParser = class {
|
|
129
|
+
parse(context) {
|
|
130
|
+
const { $, pageUrl } = context;
|
|
131
|
+
return {
|
|
132
|
+
description: $('meta[property="og:description"]').attr("content"),
|
|
133
|
+
image: resolveUrl($('meta[property="og:image"]').attr("content") ?? "", pageUrl),
|
|
134
|
+
locale: $('meta[property="og:locale"]').attr("content"),
|
|
135
|
+
siteName: $('meta[property="og:site_name"]').attr("content"),
|
|
136
|
+
title: $('meta[property="og:title"]').attr("content"),
|
|
137
|
+
type: $('meta[property="og:type"]').attr("content")
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
// src/parsers/twitter-card-parser.ts
|
|
143
|
+
var TwitterCardParser = class {
|
|
144
|
+
parse(context) {
|
|
145
|
+
const { $, pageUrl } = context;
|
|
146
|
+
return {
|
|
147
|
+
creator: $('meta[name="twitter:creator"]').attr("content"),
|
|
148
|
+
description: $('meta[name="twitter:description"]').attr("content"),
|
|
149
|
+
image: resolveUrl($('meta[name="twitter:image"]').attr("content") ?? "", pageUrl),
|
|
150
|
+
site: $('meta[name="twitter:site"]').attr("content"),
|
|
151
|
+
title: $('meta[name="twitter:title"]').attr("content")
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
// src/parsers/index.ts
|
|
157
|
+
var parsers = {
|
|
158
|
+
html: new HtmlMetaParser(),
|
|
159
|
+
openGraph: new OpenGraphParser(),
|
|
160
|
+
twitter: new TwitterCardParser()
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// src/extractors/extract-metadata.ts
|
|
164
|
+
function extractMetadata(context) {
|
|
165
|
+
return {
|
|
166
|
+
html: parsers.html.parse(context),
|
|
167
|
+
openGraph: parsers.openGraph.parse(context),
|
|
168
|
+
twitter: parsers.twitter.parse(context),
|
|
169
|
+
url: context.pageUrl
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// src/fetchers/fetch-html.ts
|
|
174
|
+
async function fetchHtml(url, options) {
|
|
175
|
+
const controller = new AbortController();
|
|
176
|
+
const timeout = setTimeout(() => controller.abort(), options.timeout ?? 5e3);
|
|
177
|
+
const response = await fetch(url, { signal: controller.signal });
|
|
178
|
+
clearTimeout(timeout);
|
|
179
|
+
return {
|
|
180
|
+
contentType: response.headers.get("content-type") ?? void 0,
|
|
181
|
+
html: await response.text(),
|
|
182
|
+
url: response.url
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// src/probe.ts
|
|
187
|
+
async function probe(url, options) {
|
|
188
|
+
try {
|
|
189
|
+
const page = await fetchHtml(url, { timeout: options?.timeout });
|
|
190
|
+
const $ = (0, import_cheerio.load)(page.html);
|
|
191
|
+
const pageUrl = page.url;
|
|
192
|
+
const context = { $, pageUrl };
|
|
193
|
+
const metadata = extractMetadata(context);
|
|
194
|
+
const diagnostics = buildDiagnostics(metadata);
|
|
195
|
+
return {
|
|
196
|
+
diagnostics,
|
|
197
|
+
metadata,
|
|
198
|
+
page: {
|
|
199
|
+
contentType: page.contentType,
|
|
200
|
+
finalUrl: page.url,
|
|
201
|
+
requestedUrl: url
|
|
202
|
+
},
|
|
203
|
+
status: "success"
|
|
204
|
+
};
|
|
205
|
+
} catch (error) {
|
|
206
|
+
if (error instanceof Error && error.name === "AbortError")
|
|
207
|
+
return {
|
|
208
|
+
error: {
|
|
209
|
+
code: "TIMEOUT",
|
|
210
|
+
message: "Request timed out"
|
|
211
|
+
},
|
|
212
|
+
status: "failed"
|
|
213
|
+
};
|
|
214
|
+
if (error instanceof TypeError)
|
|
215
|
+
return {
|
|
216
|
+
error: {
|
|
217
|
+
code: "FETCH_FAILED",
|
|
218
|
+
message: error.message
|
|
219
|
+
},
|
|
220
|
+
status: "failed"
|
|
221
|
+
};
|
|
222
|
+
return {
|
|
223
|
+
error: {
|
|
224
|
+
code: "UNKNOWN_ERROR",
|
|
225
|
+
message: "Unknown error"
|
|
226
|
+
},
|
|
227
|
+
status: "failed"
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// src/probe-many.ts
|
|
233
|
+
async function probeMany(urls, options) {
|
|
234
|
+
return Promise.all(urls.map((url) => probe(url, options)));
|
|
235
|
+
}
|
|
236
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
237
|
+
0 && (module.exports = {
|
|
238
|
+
buildDiagnostics,
|
|
239
|
+
getMetadataCoverage,
|
|
240
|
+
probe,
|
|
241
|
+
probeMany,
|
|
242
|
+
resolveMetadata
|
|
243
|
+
});
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// src/utils/extract-fields.ts
|
|
2
|
+
function extractFields(source) {
|
|
3
|
+
if (!source) {
|
|
4
|
+
return [];
|
|
5
|
+
}
|
|
6
|
+
return Object.entries(source).filter(([, value]) => value !== void 0 && value !== null && value !== "").map(([key]) => key);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
// src/build-diagnostics.ts
|
|
10
|
+
function buildDiagnostics(metadata) {
|
|
11
|
+
const openGraphFields = extractFields(metadata.openGraph);
|
|
12
|
+
const twitterFields = extractFields(metadata.twitter);
|
|
13
|
+
const htmlFields = extractFields(metadata.html);
|
|
14
|
+
return {
|
|
15
|
+
html: {
|
|
16
|
+
detected: htmlFields.length > 0,
|
|
17
|
+
fields: htmlFields
|
|
18
|
+
},
|
|
19
|
+
openGraph: {
|
|
20
|
+
detected: openGraphFields.length > 0,
|
|
21
|
+
fields: openGraphFields
|
|
22
|
+
},
|
|
23
|
+
twitter: {
|
|
24
|
+
detected: twitterFields.length > 0,
|
|
25
|
+
fields: twitterFields
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// src/resolve-metadata.ts
|
|
31
|
+
var DEFAULT_PRIORITY = ["openGraph", "twitter", "html"];
|
|
32
|
+
function resolveField(metadata, field, priority) {
|
|
33
|
+
for (const source of priority) {
|
|
34
|
+
const value = metadata[source]?.[field];
|
|
35
|
+
if (value !== void 0 && value !== null && value !== "") {
|
|
36
|
+
return value;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return void 0;
|
|
40
|
+
}
|
|
41
|
+
function resolveMetadata(metadata, options) {
|
|
42
|
+
return {
|
|
43
|
+
canonicalUrl: metadata.html?.canonicalUrl,
|
|
44
|
+
description: resolveField(
|
|
45
|
+
metadata,
|
|
46
|
+
"description",
|
|
47
|
+
options?.descriptionPriority ?? DEFAULT_PRIORITY
|
|
48
|
+
),
|
|
49
|
+
favicon: metadata.html?.favicon,
|
|
50
|
+
image: resolveField(metadata, "image", options?.imagePriority ?? DEFAULT_PRIORITY),
|
|
51
|
+
siteName: metadata.openGraph?.siteName,
|
|
52
|
+
title: resolveField(metadata, "title", options?.titlePriority ?? DEFAULT_PRIORITY)
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// src/get-metadata-coverage.ts
|
|
57
|
+
var REQUIRED_FIELDS = ["title", "description", "image", "favicon"];
|
|
58
|
+
function getMetadataCoverage(metadata) {
|
|
59
|
+
const resolved = resolveMetadata(metadata);
|
|
60
|
+
const available = REQUIRED_FIELDS.filter((field) => resolved[field]);
|
|
61
|
+
const missing = REQUIRED_FIELDS.filter((field) => !resolved[field]);
|
|
62
|
+
return {
|
|
63
|
+
available: [...available],
|
|
64
|
+
missing: [...missing],
|
|
65
|
+
score: Math.round(available.length / REQUIRED_FIELDS.length * 100)
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// src/probe.ts
|
|
70
|
+
import { load } from "cheerio";
|
|
71
|
+
|
|
72
|
+
// src/utils/resolve-url.ts
|
|
73
|
+
function resolveUrl(url, baseUrl) {
|
|
74
|
+
try {
|
|
75
|
+
return new URL(url, baseUrl).toString();
|
|
76
|
+
} catch {
|
|
77
|
+
return url;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// src/parsers/html-meta-parser.ts
|
|
82
|
+
var HtmlMetaParser = class {
|
|
83
|
+
parse(context) {
|
|
84
|
+
const { $, pageUrl } = context;
|
|
85
|
+
return {
|
|
86
|
+
canonicalUrl: $('link[rel="canonical"]').attr("href"),
|
|
87
|
+
description: $('meta[name="description"]').attr("content"),
|
|
88
|
+
favicon: resolveUrl(
|
|
89
|
+
$('link[rel="icon"]').attr("href") ?? $('link[rel="shortcut icon"]').attr("href") ?? $('link[rel="apple-touch-icon"]').attr("href") ?? "/favicon.ico",
|
|
90
|
+
pageUrl
|
|
91
|
+
),
|
|
92
|
+
title: $("title").text()
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
// src/parsers/open-graph-parser.ts
|
|
98
|
+
var OpenGraphParser = class {
|
|
99
|
+
parse(context) {
|
|
100
|
+
const { $, pageUrl } = context;
|
|
101
|
+
return {
|
|
102
|
+
description: $('meta[property="og:description"]').attr("content"),
|
|
103
|
+
image: resolveUrl($('meta[property="og:image"]').attr("content") ?? "", pageUrl),
|
|
104
|
+
locale: $('meta[property="og:locale"]').attr("content"),
|
|
105
|
+
siteName: $('meta[property="og:site_name"]').attr("content"),
|
|
106
|
+
title: $('meta[property="og:title"]').attr("content"),
|
|
107
|
+
type: $('meta[property="og:type"]').attr("content")
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
// src/parsers/twitter-card-parser.ts
|
|
113
|
+
var TwitterCardParser = class {
|
|
114
|
+
parse(context) {
|
|
115
|
+
const { $, pageUrl } = context;
|
|
116
|
+
return {
|
|
117
|
+
creator: $('meta[name="twitter:creator"]').attr("content"),
|
|
118
|
+
description: $('meta[name="twitter:description"]').attr("content"),
|
|
119
|
+
image: resolveUrl($('meta[name="twitter:image"]').attr("content") ?? "", pageUrl),
|
|
120
|
+
site: $('meta[name="twitter:site"]').attr("content"),
|
|
121
|
+
title: $('meta[name="twitter:title"]').attr("content")
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
// src/parsers/index.ts
|
|
127
|
+
var parsers = {
|
|
128
|
+
html: new HtmlMetaParser(),
|
|
129
|
+
openGraph: new OpenGraphParser(),
|
|
130
|
+
twitter: new TwitterCardParser()
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
// src/extractors/extract-metadata.ts
|
|
134
|
+
function extractMetadata(context) {
|
|
135
|
+
return {
|
|
136
|
+
html: parsers.html.parse(context),
|
|
137
|
+
openGraph: parsers.openGraph.parse(context),
|
|
138
|
+
twitter: parsers.twitter.parse(context),
|
|
139
|
+
url: context.pageUrl
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// src/fetchers/fetch-html.ts
|
|
144
|
+
async function fetchHtml(url, options) {
|
|
145
|
+
const controller = new AbortController();
|
|
146
|
+
const timeout = setTimeout(() => controller.abort(), options.timeout ?? 5e3);
|
|
147
|
+
const response = await fetch(url, { signal: controller.signal });
|
|
148
|
+
clearTimeout(timeout);
|
|
149
|
+
return {
|
|
150
|
+
contentType: response.headers.get("content-type") ?? void 0,
|
|
151
|
+
html: await response.text(),
|
|
152
|
+
url: response.url
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// src/probe.ts
|
|
157
|
+
async function probe(url, options) {
|
|
158
|
+
try {
|
|
159
|
+
const page = await fetchHtml(url, { timeout: options?.timeout });
|
|
160
|
+
const $ = load(page.html);
|
|
161
|
+
const pageUrl = page.url;
|
|
162
|
+
const context = { $, pageUrl };
|
|
163
|
+
const metadata = extractMetadata(context);
|
|
164
|
+
const diagnostics = buildDiagnostics(metadata);
|
|
165
|
+
return {
|
|
166
|
+
diagnostics,
|
|
167
|
+
metadata,
|
|
168
|
+
page: {
|
|
169
|
+
contentType: page.contentType,
|
|
170
|
+
finalUrl: page.url,
|
|
171
|
+
requestedUrl: url
|
|
172
|
+
},
|
|
173
|
+
status: "success"
|
|
174
|
+
};
|
|
175
|
+
} catch (error) {
|
|
176
|
+
if (error instanceof Error && error.name === "AbortError")
|
|
177
|
+
return {
|
|
178
|
+
error: {
|
|
179
|
+
code: "TIMEOUT",
|
|
180
|
+
message: "Request timed out"
|
|
181
|
+
},
|
|
182
|
+
status: "failed"
|
|
183
|
+
};
|
|
184
|
+
if (error instanceof TypeError)
|
|
185
|
+
return {
|
|
186
|
+
error: {
|
|
187
|
+
code: "FETCH_FAILED",
|
|
188
|
+
message: error.message
|
|
189
|
+
},
|
|
190
|
+
status: "failed"
|
|
191
|
+
};
|
|
192
|
+
return {
|
|
193
|
+
error: {
|
|
194
|
+
code: "UNKNOWN_ERROR",
|
|
195
|
+
message: "Unknown error"
|
|
196
|
+
},
|
|
197
|
+
status: "failed"
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// src/probe-many.ts
|
|
203
|
+
async function probeMany(urls, options) {
|
|
204
|
+
return Promise.all(urls.map((url) => probe(url, options)));
|
|
205
|
+
}
|
|
206
|
+
export {
|
|
207
|
+
buildDiagnostics,
|
|
208
|
+
getMetadataCoverage,
|
|
209
|
+
probe,
|
|
210
|
+
probeMany,
|
|
211
|
+
resolveMetadata
|
|
212
|
+
};
|
package/package.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@linkprobe/core",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Core metadata extraction engine for LinkProbe",
|
|
5
|
+
"main": "./dist/index.js",
|
|
6
|
+
"module": "./dist/index.mjs",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"dist"
|
|
10
|
+
],
|
|
11
|
+
"license": "MIT",
|
|
12
|
+
"author": "Hossein-i",
|
|
13
|
+
"repository": {
|
|
14
|
+
"type": "git",
|
|
15
|
+
"url": "git+https://github.com/Hossein-i/linkprobe.git",
|
|
16
|
+
"directory": "packages/core"
|
|
17
|
+
},
|
|
18
|
+
"bugs": {
|
|
19
|
+
"url": "https://github.com/Hossein-i/linkprobe/issues"
|
|
20
|
+
},
|
|
21
|
+
"homepage": "https://github.com/Hossein-i/linkprobe",
|
|
22
|
+
"dependencies": {
|
|
23
|
+
"cheerio": "^1.2.0"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"tsup": "^8.5.1",
|
|
27
|
+
"typescript": "6.0.3",
|
|
28
|
+
"vitest": "^4.1.7",
|
|
29
|
+
"@linkprobe/standard": "0.1.0"
|
|
30
|
+
},
|
|
31
|
+
"publishConfig": {
|
|
32
|
+
"access": "public"
|
|
33
|
+
},
|
|
34
|
+
"scripts": {
|
|
35
|
+
"lint": "eslint --max-warnings 0",
|
|
36
|
+
"check-types": "tsc --noEmit",
|
|
37
|
+
"build": "tsup src/index.ts --format esm,cjs --dts",
|
|
38
|
+
"test": "vitest"
|
|
39
|
+
}
|
|
40
|
+
}
|