gscdump 1.3.2 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/batch.d.mts +12 -0
- package/dist/api/batch.mjs +29 -0
- package/dist/api/indexing.d.mts +37 -0
- package/dist/api/indexing.mjs +25 -0
- package/dist/api/inspection.d.mts +89 -0
- package/dist/api/inspection.mjs +128 -0
- package/dist/api/oauth.d.mts +48 -0
- package/dist/api/oauth.mjs +154 -0
- package/dist/api/sites.d.mts +45 -0
- package/dist/api/sites.mjs +34 -0
- package/dist/api/verification.d.mts +43 -0
- package/dist/api/verification.mjs +89 -0
- package/dist/contracts.d.mts +46 -1
- package/dist/core/cli-format.mjs +9 -0
- package/dist/core/client.d.mts +122 -0
- package/dist/core/client.mjs +246 -0
- package/dist/core/errors.d.mts +66 -0
- package/dist/core/errors.mjs +207 -0
- package/dist/core/indexing-issues.d.mts +60 -0
- package/dist/core/indexing-issues.mjs +139 -0
- package/dist/core/property.d.mts +41 -0
- package/dist/core/property.mjs +74 -0
- package/dist/core/quota.d.mts +2 -0
- package/dist/core/quota.mjs +2 -0
- package/dist/core/result.d.mts +19 -1
- package/dist/core/scope-values.d.mts +7 -0
- package/dist/core/scope-values.mjs +15 -0
- package/dist/core/scopes.d.mts +5 -0
- package/dist/core/scopes.mjs +11 -0
- package/dist/core/site-url.d.mts +25 -0
- package/dist/core/site-url.mjs +30 -0
- package/dist/core/types.d.mts +201 -0
- package/dist/core/types.mjs +1 -0
- package/dist/dates.d.mts +2 -2
- package/dist/dates.mjs +2 -2
- package/dist/index.d.mts +20 -814
- package/dist/index.mjs +19 -1243
- package/dist/onboarding.d.mts +2 -0
- package/dist/onboarding.mjs +2 -0
- package/dist/{_chunks → query}/builder.d.mts +2 -2
- package/dist/query/builder.mjs +86 -0
- package/dist/query/columns.d.mts +15 -0
- package/dist/query/columns.mjs +23 -0
- package/dist/query/constants.d.mts +19 -0
- package/dist/query/constants.mjs +16 -0
- package/dist/query/errors.d.mts +85 -0
- package/dist/query/errors.mjs +141 -0
- package/dist/query/index.d.mts +9 -75
- package/dist/query/index.mjs +7 -230
- package/dist/query/operator-meta.mjs +41 -0
- package/dist/query/operators.d.mts +24 -0
- package/dist/query/operators.mjs +124 -0
- package/dist/query/plan.d.mts +92 -2
- package/dist/query/plan.mjs +3 -1
- package/dist/query/resolver.d.mts +40 -0
- package/dist/query/resolver.mjs +243 -0
- package/dist/{_chunks → query}/types.d.mts +3 -25
- package/dist/query/utils/countries.d.mts +7 -0
- package/dist/{_chunks/resolver.mjs → query/utils/countries.mjs} +1 -434
- package/dist/{_chunks → query/utils}/dayjs.mjs +2 -4
- package/dist/sitemap.d.mts +26 -0
- package/dist/sitemap.mjs +70 -0
- package/dist/url.d.mts +9 -0
- package/dist/url.mjs +6 -0
- package/package.json +2 -3
- package/dist/_chunks/contracts.d.mts +0 -47
- package/dist/_chunks/plan.d.mts +0 -175
- package/dist/_chunks/result.d.mts +0 -20
- /package/dist/{_chunks → core}/gsc-dates.d.mts +0 -0
- /package/dist/{_chunks → core}/gsc-dates.mjs +0 -0
- /package/dist/{_chunks → query/utils}/dayjs.d.mts +0 -0
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { err, ok, unwrapResult } from "../core/result.mjs";
|
|
2
|
-
import { addDays } from "./gsc-dates.mjs";
|
|
3
1
|
var countries_default = [
|
|
4
2
|
{
|
|
5
3
|
"name": "Afghanistan",
|
|
@@ -1496,435 +1494,4 @@ var countries_default = [
|
|
|
1496
1494
|
"country-code": "716"
|
|
1497
1495
|
}
|
|
1498
1496
|
];
|
|
1499
|
-
|
|
1500
|
-
MOBILE: "MOBILE",
|
|
1501
|
-
DESKTOP: "DESKTOP",
|
|
1502
|
-
TABLET: "TABLET"
|
|
1503
|
-
};
|
|
1504
|
-
const SearchTypes = {
|
|
1505
|
-
WEB: "web",
|
|
1506
|
-
IMAGE: "image",
|
|
1507
|
-
VIDEO: "video",
|
|
1508
|
-
NEWS: "news",
|
|
1509
|
-
DISCOVER: "discover",
|
|
1510
|
-
GOOGLE_NEWS: "googleNews"
|
|
1511
|
-
};
|
|
1512
|
-
const Countries = Object.fromEntries(countries_default.map((c) => [c["alpha-3"], c["alpha-3"].toLowerCase()]));
|
|
1513
|
-
const TIME_AXIS_DIMENSIONS = /* @__PURE__ */ new Set(["date", "hour"]);
|
|
1514
|
-
const queryErrors = {
|
|
1515
|
-
missingDateRange() {
|
|
1516
|
-
return {
|
|
1517
|
-
kind: "missing-date-range",
|
|
1518
|
-
message: "Date range required: use .where(between(date, start, end)) or .where(and(gte(date, start), lte(date, end)))"
|
|
1519
|
-
};
|
|
1520
|
-
},
|
|
1521
|
-
invalidRowLimit(value) {
|
|
1522
|
-
return {
|
|
1523
|
-
kind: "invalid-row-limit",
|
|
1524
|
-
value,
|
|
1525
|
-
message: `rowLimit must be a positive integer, got ${value}`
|
|
1526
|
-
};
|
|
1527
|
-
},
|
|
1528
|
-
invalidStartRow(value) {
|
|
1529
|
-
return {
|
|
1530
|
-
kind: "invalid-start-row",
|
|
1531
|
-
value,
|
|
1532
|
-
message: `startRow must be a non-negative integer, got ${value}`
|
|
1533
|
-
};
|
|
1534
|
-
},
|
|
1535
|
-
hourDimensionRequiresHourlyState() {
|
|
1536
|
-
return {
|
|
1537
|
-
kind: "invalid-data-state",
|
|
1538
|
-
message: "hour dimension requires dataState: \"hourly_all\""
|
|
1539
|
-
};
|
|
1540
|
-
},
|
|
1541
|
-
hourlyStateRequiresHourDimension() {
|
|
1542
|
-
return {
|
|
1543
|
-
kind: "invalid-data-state",
|
|
1544
|
-
message: "dataState: \"hourly_all\" requires grouping by hour dimension"
|
|
1545
|
-
};
|
|
1546
|
-
},
|
|
1547
|
-
byPropertyUnsupportedSearchType() {
|
|
1548
|
-
return {
|
|
1549
|
-
kind: "invalid-aggregation-type",
|
|
1550
|
-
message: "aggregationType: \"byProperty\" is not supported for type \"discover\" or \"googleNews\""
|
|
1551
|
-
};
|
|
1552
|
-
},
|
|
1553
|
-
byPropertyNotAllowedWithPage() {
|
|
1554
|
-
return {
|
|
1555
|
-
kind: "invalid-aggregation-type",
|
|
1556
|
-
message: "aggregationType: \"byProperty\" is not allowed when grouping or filtering by page"
|
|
1557
|
-
};
|
|
1558
|
-
},
|
|
1559
|
-
byNewsShowcaseRequiresSearchType() {
|
|
1560
|
-
return {
|
|
1561
|
-
kind: "invalid-aggregation-type",
|
|
1562
|
-
message: "aggregationType: \"byNewsShowcasePanel\" requires type \"discover\" or \"googleNews\""
|
|
1563
|
-
};
|
|
1564
|
-
},
|
|
1565
|
-
byNewsShowcaseNotAllowedWithPage() {
|
|
1566
|
-
return {
|
|
1567
|
-
kind: "invalid-aggregation-type",
|
|
1568
|
-
message: "aggregationType: \"byNewsShowcasePanel\" is not allowed when grouping or filtering by page"
|
|
1569
|
-
};
|
|
1570
|
-
},
|
|
1571
|
-
byNewsShowcaseRequiresShowcaseFilter() {
|
|
1572
|
-
return {
|
|
1573
|
-
kind: "invalid-aggregation-type",
|
|
1574
|
-
message: "aggregationType: \"byNewsShowcasePanel\" requires a searchAppearance equals \"NEWS_SHOWCASE\" filter and no other searchAppearance filter"
|
|
1575
|
-
};
|
|
1576
|
-
},
|
|
1577
|
-
invalidBuilderState(cause) {
|
|
1578
|
-
return {
|
|
1579
|
-
kind: "invalid-builder-state",
|
|
1580
|
-
cause,
|
|
1581
|
-
message: "Invalid state"
|
|
1582
|
-
};
|
|
1583
|
-
},
|
|
1584
|
-
malformedFilterLeaf() {
|
|
1585
|
-
return {
|
|
1586
|
-
kind: "invalid-filter",
|
|
1587
|
-
message: "Malformed filter: each filter leaf requires a string `dimension` and `operator`"
|
|
1588
|
-
};
|
|
1589
|
-
},
|
|
1590
|
-
unsupportedCapability(capability, context) {
|
|
1591
|
-
return {
|
|
1592
|
-
kind: "unsupported-capability",
|
|
1593
|
-
capability,
|
|
1594
|
-
context,
|
|
1595
|
-
message: `${context} requires ${capability} capability`
|
|
1596
|
-
};
|
|
1597
|
-
},
|
|
1598
|
-
unresolvableDataset(dimensions, filterDims = []) {
|
|
1599
|
-
const grouped = dimensions.filter((d) => !TIME_AXIS_DIMENSIONS.has(d));
|
|
1600
|
-
const filtered = filterDims.filter((d) => !TIME_AXIS_DIMENSIONS.has(d));
|
|
1601
|
-
return {
|
|
1602
|
-
kind: "unresolvable-dataset",
|
|
1603
|
-
dimensions,
|
|
1604
|
-
filterDims,
|
|
1605
|
-
message: `Cannot resolve a [${grouped.join(", ")}] breakdown filtered by [${filtered.join(", ")}] from stored data: these dimensions live in separate per-dimension tables. Only the live GSC API computes cross-dimension aggregates.`
|
|
1606
|
-
};
|
|
1607
|
-
}
|
|
1608
|
-
};
|
|
1609
|
-
const QUERY_ERROR_KINDS = /* @__PURE__ */ new Set([
|
|
1610
|
-
"missing-date-range",
|
|
1611
|
-
"invalid-row-limit",
|
|
1612
|
-
"invalid-start-row",
|
|
1613
|
-
"invalid-data-state",
|
|
1614
|
-
"invalid-aggregation-type",
|
|
1615
|
-
"invalid-builder-state",
|
|
1616
|
-
"invalid-filter",
|
|
1617
|
-
"unsupported-capability",
|
|
1618
|
-
"unresolvable-dataset"
|
|
1619
|
-
]);
|
|
1620
|
-
function isQueryError(value) {
|
|
1621
|
-
return typeof value === "object" && value !== null && QUERY_ERROR_KINDS.has(value.kind) && typeof value.message === "string";
|
|
1622
|
-
}
|
|
1623
|
-
var UnsupportedLogicalCapabilityError = class extends Error {
|
|
1624
|
-
queryError;
|
|
1625
|
-
constructor(capability, context) {
|
|
1626
|
-
const error = queryErrors.unsupportedCapability(capability, context);
|
|
1627
|
-
super(error.message);
|
|
1628
|
-
this.name = "UnsupportedLogicalCapabilityError";
|
|
1629
|
-
this.queryError = error;
|
|
1630
|
-
}
|
|
1631
|
-
};
|
|
1632
|
-
var UnresolvableDatasetError = class extends Error {
|
|
1633
|
-
queryError;
|
|
1634
|
-
constructor(dimensions, filterDims = []) {
|
|
1635
|
-
const error = queryErrors.unresolvableDataset(dimensions, filterDims);
|
|
1636
|
-
super(error.message);
|
|
1637
|
-
this.name = "UnresolvableDatasetError";
|
|
1638
|
-
this.queryError = error;
|
|
1639
|
-
}
|
|
1640
|
-
};
|
|
1641
|
-
function queryErrorToException(error) {
|
|
1642
|
-
switch (error.kind) {
|
|
1643
|
-
case "unsupported-capability": return new UnsupportedLogicalCapabilityError(error.capability, error.context);
|
|
1644
|
-
case "unresolvable-dataset": return new UnresolvableDatasetError(error.dimensions, error.filterDims);
|
|
1645
|
-
default: {
|
|
1646
|
-
const exception = new Error(error.message);
|
|
1647
|
-
if ("cause" in error && error.cause !== void 0) exception.cause = error.cause;
|
|
1648
|
-
exception.queryError = error;
|
|
1649
|
-
return exception;
|
|
1650
|
-
}
|
|
1651
|
-
}
|
|
1652
|
-
}
|
|
1653
|
-
const DATE_OPERATORS = [
|
|
1654
|
-
"gte",
|
|
1655
|
-
"gt",
|
|
1656
|
-
"lte",
|
|
1657
|
-
"lt",
|
|
1658
|
-
"between"
|
|
1659
|
-
];
|
|
1660
|
-
const METRIC_OPERATORS = [
|
|
1661
|
-
"metricGte",
|
|
1662
|
-
"metricGt",
|
|
1663
|
-
"metricLte",
|
|
1664
|
-
"metricLt",
|
|
1665
|
-
"metricBetween"
|
|
1666
|
-
];
|
|
1667
|
-
const SPECIAL_OPERATORS = ["topLevel"];
|
|
1668
|
-
const QUERY_PARAMS = ["searchType"];
|
|
1669
|
-
const FILTER_METRICS = [
|
|
1670
|
-
"clicks",
|
|
1671
|
-
"impressions",
|
|
1672
|
-
"ctr",
|
|
1673
|
-
"position"
|
|
1674
|
-
];
|
|
1675
|
-
function isDateOperator(op) {
|
|
1676
|
-
return DATE_OPERATORS.includes(op);
|
|
1677
|
-
}
|
|
1678
|
-
function isMetricOperator(op) {
|
|
1679
|
-
return METRIC_OPERATORS.includes(op);
|
|
1680
|
-
}
|
|
1681
|
-
function isSpecialOperator(op) {
|
|
1682
|
-
return SPECIAL_OPERATORS.includes(op);
|
|
1683
|
-
}
|
|
1684
|
-
function isQueryParam(value) {
|
|
1685
|
-
return QUERY_PARAMS.includes(value);
|
|
1686
|
-
}
|
|
1687
|
-
function isMetric(value) {
|
|
1688
|
-
return FILTER_METRICS.includes(value);
|
|
1689
|
-
}
|
|
1690
|
-
function isRegexOperator(op) {
|
|
1691
|
-
return op === "includingRegex" || op === "excludingRegex";
|
|
1692
|
-
}
|
|
1693
|
-
const KNOWN_SEARCH_TYPES = new Set(Object.values(SearchTypes));
|
|
1694
|
-
function isWireGroupType(type) {
|
|
1695
|
-
return type === "and" || type === "or";
|
|
1696
|
-
}
|
|
1697
|
-
function convertWireLeaf(alt) {
|
|
1698
|
-
if (!alt.column || !alt.type || isWireGroupType(alt.type)) return null;
|
|
1699
|
-
const f = {
|
|
1700
|
-
dimension: alt.column,
|
|
1701
|
-
operator: alt.type,
|
|
1702
|
-
expression: alt.type === "between" ? alt.from ?? "" : alt.value ?? ""
|
|
1703
|
-
};
|
|
1704
|
-
if (alt.type === "between" && alt.to) f.expression2 = alt.to;
|
|
1705
|
-
return f;
|
|
1706
|
-
}
|
|
1707
|
-
function convertWireGroup(alt) {
|
|
1708
|
-
if (!isWireGroupType(alt.type)) {
|
|
1709
|
-
const leaf = convertWireLeaf(alt);
|
|
1710
|
-
return leaf ? { _filters: [leaf] } : null;
|
|
1711
|
-
}
|
|
1712
|
-
const leaves = [];
|
|
1713
|
-
const nested = [];
|
|
1714
|
-
for (const child of alt.filters ?? []) if (isWireGroupType(child.type)) {
|
|
1715
|
-
const sub = convertWireGroup(child);
|
|
1716
|
-
if (sub) nested.push(sub);
|
|
1717
|
-
} else {
|
|
1718
|
-
const leaf = convertWireLeaf(child);
|
|
1719
|
-
if (leaf) leaves.push(leaf);
|
|
1720
|
-
}
|
|
1721
|
-
if (leaves.length === 0 && nested.length === 0) return null;
|
|
1722
|
-
return {
|
|
1723
|
-
_filters: leaves,
|
|
1724
|
-
_nestedGroups: nested.length > 0 ? nested : void 0,
|
|
1725
|
-
_groupType: alt.type
|
|
1726
|
-
};
|
|
1727
|
-
}
|
|
1728
|
-
function isWireFilter(input) {
|
|
1729
|
-
if (!input || typeof input !== "object") return false;
|
|
1730
|
-
const o = input;
|
|
1731
|
-
if ("_filters" in o) return false;
|
|
1732
|
-
return "type" in o && typeof o.type === "string" || "filters" in o && Array.isArray(o.filters);
|
|
1733
|
-
}
|
|
1734
|
-
function normalizeFilter(input) {
|
|
1735
|
-
if (!input) return void 0;
|
|
1736
|
-
if (isWireFilter(input)) return convertWireGroup(input) ?? void 0;
|
|
1737
|
-
if (typeof input === "object" && Array.isArray(input._filters)) return input;
|
|
1738
|
-
}
|
|
1739
|
-
function normalizeOrderBy(orderBy) {
|
|
1740
|
-
const spec = Array.isArray(orderBy) ? orderBy[0] : orderBy;
|
|
1741
|
-
if (!spec || typeof spec !== "object") return void 0;
|
|
1742
|
-
const o = spec;
|
|
1743
|
-
if (typeof o.column !== "string" || o.column.length === 0) return void 0;
|
|
1744
|
-
const dir = typeof o.dir === "string" ? o.dir.toLowerCase() === "asc" ? "asc" : "desc" : o.desc === false ? "asc" : "desc";
|
|
1745
|
-
return {
|
|
1746
|
-
column: o.column,
|
|
1747
|
-
dir
|
|
1748
|
-
};
|
|
1749
|
-
}
|
|
1750
|
-
function hasMalformedFilterLeaf(filter) {
|
|
1751
|
-
if (!filter || typeof filter !== "object") return false;
|
|
1752
|
-
if (Array.isArray(filter._filters)) {
|
|
1753
|
-
for (const leaf of filter._filters) if (!leaf || typeof leaf !== "object" || typeof leaf.operator !== "string" || typeof leaf.dimension !== "string") return true;
|
|
1754
|
-
}
|
|
1755
|
-
if (Array.isArray(filter._nestedGroups)) {
|
|
1756
|
-
for (const group of filter._nestedGroups) if (hasMalformedFilterLeaf(group)) return true;
|
|
1757
|
-
}
|
|
1758
|
-
return false;
|
|
1759
|
-
}
|
|
1760
|
-
function normalizeBuilderStateResult(state) {
|
|
1761
|
-
if (!state || typeof state !== "object") return err(queryErrors.invalidBuilderState(state));
|
|
1762
|
-
const s = state;
|
|
1763
|
-
const filter = normalizeFilter(s.filter);
|
|
1764
|
-
if (hasMalformedFilterLeaf(filter)) return err(queryErrors.malformedFilterLeaf());
|
|
1765
|
-
const normalized = {
|
|
1766
|
-
dimensions: Array.isArray(s.dimensions) ? s.dimensions : [],
|
|
1767
|
-
metrics: s.metrics,
|
|
1768
|
-
filter,
|
|
1769
|
-
orderBy: normalizeOrderBy(s.orderBy),
|
|
1770
|
-
rowLimit: s.rowLimit,
|
|
1771
|
-
startRow: s.startRow,
|
|
1772
|
-
dataState: s.dataState,
|
|
1773
|
-
aggregationType: s.aggregationType
|
|
1774
|
-
};
|
|
1775
|
-
if (typeof s.searchType === "string" && KNOWN_SEARCH_TYPES.has(s.searchType)) normalized.searchType = s.searchType;
|
|
1776
|
-
return ok(normalized);
|
|
1777
|
-
}
|
|
1778
|
-
function normalizeBuilderState(state) {
|
|
1779
|
-
return unwrapResult(normalizeBuilderStateResult(state), queryErrorToException);
|
|
1780
|
-
}
|
|
1781
|
-
function extractSpecialFilters(filter) {
|
|
1782
|
-
if (!filter || !Array.isArray(filter._filters)) return {};
|
|
1783
|
-
let startDate;
|
|
1784
|
-
let endDate;
|
|
1785
|
-
let searchType;
|
|
1786
|
-
const otherFilters = [];
|
|
1787
|
-
const cleanedNestedGroups = [];
|
|
1788
|
-
for (const f of filter._filters) if (f.dimension === "date" && isDateOperator(f.operator)) switch (f.operator) {
|
|
1789
|
-
case "gte":
|
|
1790
|
-
startDate = f.expression;
|
|
1791
|
-
break;
|
|
1792
|
-
case "gt":
|
|
1793
|
-
startDate = addDays(f.expression, 1);
|
|
1794
|
-
break;
|
|
1795
|
-
case "lte":
|
|
1796
|
-
endDate = f.expression;
|
|
1797
|
-
break;
|
|
1798
|
-
case "lt":
|
|
1799
|
-
endDate = addDays(f.expression, -1);
|
|
1800
|
-
break;
|
|
1801
|
-
case "between":
|
|
1802
|
-
startDate = f.expression;
|
|
1803
|
-
endDate = f.expression2;
|
|
1804
|
-
break;
|
|
1805
|
-
}
|
|
1806
|
-
else if (isQueryParam(f.dimension)) {
|
|
1807
|
-
if (f.dimension === "searchType") searchType = f.expression;
|
|
1808
|
-
} else if (isMetricOperator(f.operator) || isSpecialOperator(f.operator)) otherFilters.push(f);
|
|
1809
|
-
else otherFilters.push(f);
|
|
1810
|
-
if (filter._nestedGroups) for (const nested of filter._nestedGroups) {
|
|
1811
|
-
const extracted = extractSpecialFilters(nested);
|
|
1812
|
-
if (!startDate && extracted.startDate) startDate = extracted.startDate;
|
|
1813
|
-
if (!endDate && extracted.endDate) endDate = extracted.endDate;
|
|
1814
|
-
if (!searchType && extracted.searchType) searchType = extracted.searchType;
|
|
1815
|
-
if (extracted.dimensionFilter) cleanedNestedGroups.push(extracted.dimensionFilter);
|
|
1816
|
-
}
|
|
1817
|
-
const dimensionFilter = otherFilters.length > 0 || cleanedNestedGroups.length > 0 ? {
|
|
1818
|
-
...filter,
|
|
1819
|
-
_filters: otherFilters,
|
|
1820
|
-
_nestedGroups: cleanedNestedGroups.length > 0 ? cleanedNestedGroups : void 0
|
|
1821
|
-
} : void 0;
|
|
1822
|
-
return {
|
|
1823
|
-
startDate,
|
|
1824
|
-
endDate,
|
|
1825
|
-
searchType,
|
|
1826
|
-
dimensionFilter
|
|
1827
|
-
};
|
|
1828
|
-
}
|
|
1829
|
-
function extractDateRange(input) {
|
|
1830
|
-
const { startDate, endDate } = extractSpecialFilters(normalizeFilter(input));
|
|
1831
|
-
return {
|
|
1832
|
-
startDate,
|
|
1833
|
-
endDate
|
|
1834
|
-
};
|
|
1835
|
-
}
|
|
1836
|
-
function extractMetricFilters(input) {
|
|
1837
|
-
const filter = normalizeFilter(input);
|
|
1838
|
-
if (!filter) return [];
|
|
1839
|
-
const metricFilters = filter._filters.filter((f) => isMetricOperator(f.operator));
|
|
1840
|
-
const nested = filter._nestedGroups?.flatMap((g) => extractMetricFilters(g)) ?? [];
|
|
1841
|
-
return [...metricFilters, ...nested];
|
|
1842
|
-
}
|
|
1843
|
-
function extractSpecialOperatorFilters(input) {
|
|
1844
|
-
const filter = normalizeFilter(input);
|
|
1845
|
-
if (!filter) return [];
|
|
1846
|
-
const special = filter._filters.filter((f) => isSpecialOperator(f.operator));
|
|
1847
|
-
const nested = filter._nestedGroups?.flatMap((g) => extractSpecialOperatorFilters(g)) ?? [];
|
|
1848
|
-
return [...special, ...nested];
|
|
1849
|
-
}
|
|
1850
|
-
function extractSearchType(state) {
|
|
1851
|
-
if (!state) return void 0;
|
|
1852
|
-
if (state.searchType && KNOWN_SEARCH_TYPES.has(state.searchType)) return state.searchType;
|
|
1853
|
-
const filter = state.filter;
|
|
1854
|
-
const raw = extractSpecialFilters(normalizeFilter(filter)).searchType;
|
|
1855
|
-
if (typeof raw !== "string" || raw.length === 0) return void 0;
|
|
1856
|
-
return KNOWN_SEARCH_TYPES.has(raw) ? raw : void 0;
|
|
1857
|
-
}
|
|
1858
|
-
function resolveToBodyResult(state) {
|
|
1859
|
-
const { startDate, endDate, searchType, dimensionFilter } = extractSpecialFilters(state.filter);
|
|
1860
|
-
if (!startDate || !endDate) return err(queryErrors.missingDateRange());
|
|
1861
|
-
const body = {
|
|
1862
|
-
dimensions: state.dimensions,
|
|
1863
|
-
startDate,
|
|
1864
|
-
endDate
|
|
1865
|
-
};
|
|
1866
|
-
const resolvedType = state.searchType ?? searchType;
|
|
1867
|
-
if (resolvedType) body.type = resolvedType;
|
|
1868
|
-
if (state.rowLimit !== void 0) {
|
|
1869
|
-
if (!Number.isInteger(state.rowLimit) || state.rowLimit < 1) return err(queryErrors.invalidRowLimit(state.rowLimit));
|
|
1870
|
-
body.rowLimit = state.rowLimit;
|
|
1871
|
-
}
|
|
1872
|
-
if (state.startRow !== void 0) {
|
|
1873
|
-
if (!Number.isInteger(state.startRow) || state.startRow < 0) return err(queryErrors.invalidStartRow(state.startRow));
|
|
1874
|
-
if (state.startRow > 0) body.startRow = state.startRow;
|
|
1875
|
-
}
|
|
1876
|
-
const hasHour = state.dimensions?.includes("hour");
|
|
1877
|
-
if (hasHour && state.dataState !== "hourly_all") return err(queryErrors.hourDimensionRequiresHourlyState());
|
|
1878
|
-
if (state.dataState === "hourly_all" && !hasHour) return err(queryErrors.hourlyStateRequiresHourDimension());
|
|
1879
|
-
if (state.dataState) body.dataState = state.dataState;
|
|
1880
|
-
const filterGroups = resolveFilter(dimensionFilter);
|
|
1881
|
-
if (filterGroups.length > 0) body.dimensionFilterGroups = filterGroups;
|
|
1882
|
-
if (state.aggregationType) {
|
|
1883
|
-
const groupsByPage = (body.dimensions ?? []).includes("page");
|
|
1884
|
-
const apiLeafFilters = filterGroups.flatMap((g) => g.filters ?? []);
|
|
1885
|
-
const filtersByPage = apiLeafFilters.some((f) => f.dimension === "page");
|
|
1886
|
-
if (state.aggregationType === "byProperty") {
|
|
1887
|
-
if (body.type === "discover" || body.type === "googleNews") return err(queryErrors.byPropertyUnsupportedSearchType());
|
|
1888
|
-
if (groupsByPage || filtersByPage) return err(queryErrors.byPropertyNotAllowedWithPage());
|
|
1889
|
-
}
|
|
1890
|
-
if (state.aggregationType === "byNewsShowcasePanel") {
|
|
1891
|
-
if (body.type !== "discover" && body.type !== "googleNews") return err(queryErrors.byNewsShowcaseRequiresSearchType());
|
|
1892
|
-
if (groupsByPage || filtersByPage) return err(queryErrors.byNewsShowcaseNotAllowedWithPage());
|
|
1893
|
-
const saFilters = apiLeafFilters.filter((f) => f.dimension === "searchAppearance");
|
|
1894
|
-
const hasNewsShowcase = saFilters.some((f) => f.operator === "equals" && f.expression === "NEWS_SHOWCASE");
|
|
1895
|
-
const hasOther = saFilters.some((f) => !(f.operator === "equals" && f.expression === "NEWS_SHOWCASE"));
|
|
1896
|
-
if (!hasNewsShowcase || hasOther) return err(queryErrors.byNewsShowcaseRequiresShowcaseFilter());
|
|
1897
|
-
}
|
|
1898
|
-
body.aggregationType = state.aggregationType;
|
|
1899
|
-
}
|
|
1900
|
-
return ok(body);
|
|
1901
|
-
}
|
|
1902
|
-
function resolveToBody(state) {
|
|
1903
|
-
return unwrapResult(resolveToBodyResult(state), queryErrorToException);
|
|
1904
|
-
}
|
|
1905
|
-
function isApiFilter(f) {
|
|
1906
|
-
return !isMetricOperator(f.operator) && !isSpecialOperator(f.operator);
|
|
1907
|
-
}
|
|
1908
|
-
function resolveFilter(filter) {
|
|
1909
|
-
if (!filter) return [];
|
|
1910
|
-
const groups = [];
|
|
1911
|
-
const groupType = filter._groupType ?? "and";
|
|
1912
|
-
const apiFilters = filter._filters.filter(isApiFilter);
|
|
1913
|
-
if (groupType === "or") {
|
|
1914
|
-
if (apiFilters.length > 0) groups.push({
|
|
1915
|
-
groupType: "or",
|
|
1916
|
-
filters: apiFilters.map((f) => ({
|
|
1917
|
-
dimension: f.dimension,
|
|
1918
|
-
operator: f.operator,
|
|
1919
|
-
expression: f.expression
|
|
1920
|
-
}))
|
|
1921
|
-
});
|
|
1922
|
-
} else if (apiFilters.length > 0) groups.push({ filters: apiFilters.map((f) => ({
|
|
1923
|
-
dimension: f.dimension,
|
|
1924
|
-
operator: f.operator,
|
|
1925
|
-
expression: f.expression
|
|
1926
|
-
})) });
|
|
1927
|
-
if (filter._nestedGroups) for (const nested of filter._nestedGroups) groups.push(...resolveFilter(nested));
|
|
1928
|
-
return groups;
|
|
1929
|
-
}
|
|
1930
|
-
export { Countries, DATE_OPERATORS, Devices, SearchTypes, UnresolvableDatasetError, UnsupportedLogicalCapabilityError, extractDateRange, extractMetricFilters, extractSearchType, extractSpecialOperatorFilters, isDateOperator, isMetric, isQueryError, isQueryParam, isRegexOperator, normalizeBuilderState, normalizeBuilderStateResult, normalizeFilter, queryErrorToException, queryErrors, resolveToBody, resolveToBodyResult };
|
|
1497
|
+
export { countries_default as default };
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { subDays } from "date-fns/subDays";
|
|
1
|
+
import { addDays } from "../../core/gsc-dates.mjs";
|
|
3
2
|
let pstFormatter;
|
|
4
3
|
function getPstFormatter() {
|
|
5
4
|
pstFormatter ??= new Intl.DateTimeFormat("en-CA", {
|
|
@@ -24,7 +23,6 @@ function currentPstDate() {
|
|
|
24
23
|
return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
|
|
25
24
|
}
|
|
26
25
|
function daysAgoPst(n) {
|
|
27
|
-
|
|
28
|
-
return format(subDays(new Date(year, month - 1, day, 12), n), "yyyy-MM-dd");
|
|
26
|
+
return addDays(currentPstDate(), -n);
|
|
29
27
|
}
|
|
30
28
|
export { currentPstDate, daysAgoPst };
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
interface DiscoverSitemapOptions {
|
|
2
|
+
/** User-Agent sent on the discovery requests. */
|
|
3
|
+
userAgent?: string;
|
|
4
|
+
/** AbortSignal threaded through fetches; defaults to a 10s timeout per call. */
|
|
5
|
+
signal?: AbortSignal;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Try to discover a sitemap for `domain` by checking robots.txt for a
|
|
9
|
+
* `Sitemap:` directive, then a small set of common paths. Returns the first
|
|
10
|
+
* URL that responds with a 2xx, or `null`.
|
|
11
|
+
*/
|
|
12
|
+
declare function discoverSitemap(domain: string, options?: DiscoverSitemapOptions): Promise<string | null>;
|
|
13
|
+
interface FetchSitemapUrlsOptions extends DiscoverSitemapOptions {
|
|
14
|
+
/** Maximum nested sitemap-index depth to follow. Default 3. */
|
|
15
|
+
maxDepth?: number;
|
|
16
|
+
/** Stop after this many URLs (across all nested sitemaps). Default unlimited. */
|
|
17
|
+
limit?: number;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Fetch a sitemap (or sitemap index) and return the list of `<loc>` URLs.
|
|
21
|
+
* Sitemap-index files are followed up to `maxDepth` levels. Duplicates are
|
|
22
|
+
* de-duplicated. The XML parser is regex-based — it handles the common
|
|
23
|
+
* `<loc>https://...</loc>` shape but doesn't validate the schema.
|
|
24
|
+
*/
|
|
25
|
+
declare function fetchSitemapUrls(sitemapUrl: string, options?: FetchSitemapUrlsOptions): Promise<string[]>;
|
|
26
|
+
export { DiscoverSitemapOptions, FetchSitemapUrlsOptions, discoverSitemap, fetchSitemapUrls };
|
package/dist/sitemap.mjs
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
const FETCH_TIMEOUT_MS = 1e4;
|
|
2
|
+
const COMMON_PATHS = ["/sitemap.xml", "/sitemap_index.xml"];
|
|
3
|
+
const SITEMAP_DIRECTIVE_RE = /^Sitemap:\s*(\S+)/im;
|
|
4
|
+
async function discoverSitemap(domain, options = {}) {
|
|
5
|
+
const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
|
|
6
|
+
const baseUrl = `https://${domain}`;
|
|
7
|
+
const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
|
|
8
|
+
const robotsRes = await fetch(`${baseUrl}/robots.txt`, {
|
|
9
|
+
headers: { "User-Agent": userAgent },
|
|
10
|
+
signal: signalFor()
|
|
11
|
+
}).catch(() => null);
|
|
12
|
+
if (robotsRes?.ok) {
|
|
13
|
+
const match = (await robotsRes.text()).match(SITEMAP_DIRECTIVE_RE);
|
|
14
|
+
if (match?.[1]) {
|
|
15
|
+
if ((await fetch(match[1], {
|
|
16
|
+
method: "HEAD",
|
|
17
|
+
signal: signalFor()
|
|
18
|
+
}).catch(() => null))?.ok) return match[1];
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
for (const path of COMMON_PATHS) {
|
|
22
|
+
const url = `${baseUrl}${path}`;
|
|
23
|
+
if ((await fetch(url, {
|
|
24
|
+
method: "HEAD",
|
|
25
|
+
headers: { "User-Agent": userAgent },
|
|
26
|
+
signal: signalFor()
|
|
27
|
+
}).catch(() => null))?.ok) return url;
|
|
28
|
+
}
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
const LOC_RE = /<loc>([^<]+)<\/loc>/gi;
|
|
32
|
+
const SITEMAPINDEX_RE = /<sitemapindex\b/i;
|
|
33
|
+
async function fetchSitemapUrls(sitemapUrl, options = {}) {
|
|
34
|
+
const userAgent = options.userAgent ?? "gscdump sitemap fetcher";
|
|
35
|
+
const maxDepth = options.maxDepth ?? 3;
|
|
36
|
+
const limit = options.limit;
|
|
37
|
+
const signalFor = () => options.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
|
|
38
|
+
const seen = /* @__PURE__ */ new Set();
|
|
39
|
+
const out = [];
|
|
40
|
+
const visit = async (url, depth) => {
|
|
41
|
+
if (limit != null && out.length >= limit) return;
|
|
42
|
+
if (depth > maxDepth) return;
|
|
43
|
+
const res = await fetch(url, {
|
|
44
|
+
headers: { "User-Agent": userAgent },
|
|
45
|
+
signal: signalFor()
|
|
46
|
+
});
|
|
47
|
+
if (!res.ok) throw new Error(`Fetch ${url} failed: ${res.status}`);
|
|
48
|
+
const text = await res.text();
|
|
49
|
+
if (SITEMAPINDEX_RE.test(text)) {
|
|
50
|
+
for (const match of text.matchAll(LOC_RE)) {
|
|
51
|
+
if (limit != null && out.length >= limit) return;
|
|
52
|
+
const child = match[1].trim();
|
|
53
|
+
if (!child) continue;
|
|
54
|
+
await visit(child, depth + 1);
|
|
55
|
+
}
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
for (const match of text.matchAll(LOC_RE)) {
|
|
59
|
+
const u = match[1].trim();
|
|
60
|
+
if (!u) continue;
|
|
61
|
+
if (seen.has(u)) continue;
|
|
62
|
+
seen.add(u);
|
|
63
|
+
out.push(u);
|
|
64
|
+
if (limit != null && out.length >= limit) return;
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
await visit(sitemapUrl, 0);
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
export { discoverSitemap, fetchSitemapUrls };
|
package/dist/url.d.mts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Equivalence key for matching two URLs that point at the same resource
|
|
3
|
+
* across protocol (http/https), www. prefix, trailing slash, and casing.
|
|
4
|
+
* Returns `null` for unparseable input.
|
|
5
|
+
*
|
|
6
|
+
* Example: `https://www.Example.com/Foo/` → `example.com/foo`
|
|
7
|
+
*/
|
|
8
|
+
declare function urlMatchKey(url: string): string | null;
|
|
9
|
+
export { urlMatchKey };
|
package/dist/url.mjs
ADDED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gscdump",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.4.1",
|
|
5
5
|
"description": "Direct Google Search Console client with typed queries, streaming pagination, URL inspection, and indexing helpers",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -82,9 +82,8 @@
|
|
|
82
82
|
"node": ">=22"
|
|
83
83
|
},
|
|
84
84
|
"dependencies": {
|
|
85
|
-
"date-fns": "^4.4.0",
|
|
86
85
|
"ofetch": "^1.5.1",
|
|
87
|
-
"@gscdump/contracts": "^1.
|
|
86
|
+
"@gscdump/contracts": "^1.4.1"
|
|
88
87
|
},
|
|
89
88
|
"scripts": {
|
|
90
89
|
"dev": "obuild --stub",
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { ColumnDef, ColumnType, Row, TableName, TableSchema, TenantCtx } from "@gscdump/contracts";
|
|
2
|
-
type GscSearchAnalyticsDimension = 'page' | 'query' | 'country' | 'device' | 'date' | 'hour' | 'searchAppearance';
|
|
3
|
-
type GscDataState = 'final' | 'all' | 'hourly_all';
|
|
4
|
-
interface GscSearchAnalyticsMetadata {
|
|
5
|
-
/** First date (YYYY-MM-DD, PT) still being collected. Populated when dataState=`all` and grouped by `date`. */
|
|
6
|
-
first_incomplete_date?: string;
|
|
7
|
-
/** First hour (YYYY-MM-DDThh:mm:ss±hh:mm, PT) still being collected. Populated when dataState=`hourly_all` and grouped by `hour`. */
|
|
8
|
-
first_incomplete_hour?: string;
|
|
9
|
-
}
|
|
10
|
-
type GscSearchAnalyticsFilterOperator = 'equals' | 'notEquals' | 'contains' | 'notContains' | 'includingRegex' | 'excludingRegex';
|
|
11
|
-
interface GscSearchAnalyticsFilter {
|
|
12
|
-
dimension: GscSearchAnalyticsDimension;
|
|
13
|
-
expression: string;
|
|
14
|
-
operator?: GscSearchAnalyticsFilterOperator;
|
|
15
|
-
}
|
|
16
|
-
interface GscSearchAnalyticsFilterGroup {
|
|
17
|
-
groupType?: 'and' | 'or';
|
|
18
|
-
filters: GscSearchAnalyticsFilter[];
|
|
19
|
-
}
|
|
20
|
-
type GscSearchType = 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews';
|
|
21
|
-
type GscAggregationType = 'auto' | 'byPage' | 'byProperty' | 'byNewsShowcasePanel';
|
|
22
|
-
type GscResponseAggregationType = 'auto' | 'byPage' | 'byProperty' | 'byNewsShowcasePanel';
|
|
23
|
-
interface GscSearchAnalyticsRequest {
|
|
24
|
-
startDate: string;
|
|
25
|
-
endDate: string;
|
|
26
|
-
dimensions?: GscSearchAnalyticsDimension[];
|
|
27
|
-
dimensionFilterGroups?: GscSearchAnalyticsFilterGroup[];
|
|
28
|
-
rowLimit?: number;
|
|
29
|
-
startRow?: number;
|
|
30
|
-
/** GSC search corpus. Maps to wire field `type` (the API still accepts the deprecated `searchType` alias). */
|
|
31
|
-
type?: GscSearchType;
|
|
32
|
-
dataState?: GscDataState;
|
|
33
|
-
aggregationType?: GscAggregationType;
|
|
34
|
-
}
|
|
35
|
-
interface GscSearchAnalyticsRow {
|
|
36
|
-
keys: string[];
|
|
37
|
-
clicks: number;
|
|
38
|
-
impressions: number;
|
|
39
|
-
ctr: number;
|
|
40
|
-
position: number;
|
|
41
|
-
}
|
|
42
|
-
interface GscSearchAnalyticsResponse {
|
|
43
|
-
rows?: GscSearchAnalyticsRow[];
|
|
44
|
-
responseAggregationType?: GscResponseAggregationType;
|
|
45
|
-
metadata?: GscSearchAnalyticsMetadata;
|
|
46
|
-
}
|
|
47
|
-
export { type ColumnDef, type ColumnType, GscAggregationType, GscDataState, GscResponseAggregationType, GscSearchAnalyticsDimension, GscSearchAnalyticsFilter, GscSearchAnalyticsFilterGroup, GscSearchAnalyticsFilterOperator, GscSearchAnalyticsMetadata, GscSearchAnalyticsRequest, GscSearchAnalyticsResponse, GscSearchAnalyticsRow, GscSearchType, type Row, type TableName, type TableSchema, type TenantCtx };
|