doc-fetch-cli 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +193 -0
- package/SECURITY.md +84 -0
- package/bin/doc-fetch.js +37 -0
- package/bin/install.js +171 -0
- package/cmd/docfetch/main.go +54 -0
- package/dist/doc_fetch-1.0.1-py3-none-any.whl +0 -0
- package/dist/doc_fetch-1.0.1.tar.gz +0 -0
- package/doc-fetch +0 -0
- package/doc-fetch_darwin_amd64 +0 -0
- package/doc-fetch_linux_amd64 +0 -0
- package/doc-fetch_windows_amd64.exe +0 -0
- package/doc_fetch/__init__.py +6 -0
- package/doc_fetch/__main__.py +7 -0
- package/doc_fetch/cli.py +113 -0
- package/doc_fetch.egg-info/PKG-INFO +224 -0
- package/doc_fetch.egg-info/SOURCES.txt +19 -0
- package/doc_fetch.egg-info/dependency_links.txt +1 -0
- package/doc_fetch.egg-info/entry_points.txt +2 -0
- package/doc_fetch.egg-info/not-zip-safe +1 -0
- package/doc_fetch.egg-info/top_level.txt +1 -0
- package/docs/usage.md +67 -0
- package/examples/golang-example.sh +12 -0
- package/go.mod +11 -0
- package/go.sum +38 -0
- package/package.json +18 -0
- package/pkg/fetcher/classifier.go +50 -0
- package/pkg/fetcher/describer.go +61 -0
- package/pkg/fetcher/fetcher.go +332 -0
- package/pkg/fetcher/html2md.go +71 -0
- package/pkg/fetcher/llmtxt.go +36 -0
- package/pkg/fetcher/validator.go +109 -0
- package/pkg/fetcher/writer.go +32 -0
- package/pyproject.toml +37 -0
- package/setup.py +158 -0
package/setup.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DocFetch - Dynamic documentation fetching CLI
|
|
3
|
+
Converts entire documentation sites to single markdown files for AI/LLM consumption.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
import platform
|
|
9
|
+
from setuptools import setup, find_packages
|
|
10
|
+
from setuptools.command.install import install
|
|
11
|
+
import urllib.request
|
|
12
|
+
import tarfile
|
|
13
|
+
import zipfile
|
|
14
|
+
import stat
|
|
15
|
+
import shutil
|
|
16
|
+
|
|
17
|
+
# Determine binary name based on platform
|
|
18
|
+
def get_binary_name():
|
|
19
|
+
system = platform.system().lower()
|
|
20
|
+
machine = platform.machine().lower()
|
|
21
|
+
|
|
22
|
+
# Map machine architectures
|
|
23
|
+
arch_map = {
|
|
24
|
+
'x86_64': 'amd64',
|
|
25
|
+
'amd64': 'amd64',
|
|
26
|
+
'arm64': 'arm64',
|
|
27
|
+
'aarch64': 'arm64'
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
arch = arch_map.get(machine, 'amd64')
|
|
31
|
+
|
|
32
|
+
if system == 'windows':
|
|
33
|
+
return f'doc-fetch_windows_{arch}.exe'
|
|
34
|
+
elif system == 'darwin':
|
|
35
|
+
return f'doc-fetch_darwin_{arch}'
|
|
36
|
+
else:
|
|
37
|
+
return f'doc-fetch_linux_{arch}'
|
|
38
|
+
|
|
39
|
+
class InstallCommand(install):
|
|
40
|
+
"""Custom install command to download and install the Go binary."""
|
|
41
|
+
|
|
42
|
+
def run(self):
|
|
43
|
+
# Run the standard install first
|
|
44
|
+
install.run(self)
|
|
45
|
+
|
|
46
|
+
# Download and install the Go binary
|
|
47
|
+
self.download_binary()
|
|
48
|
+
|
|
49
|
+
def download_binary(self):
|
|
50
|
+
"""Download the appropriate Go binary for the current platform."""
|
|
51
|
+
binary_name = get_binary_name()
|
|
52
|
+
download_url = f"https://github.com/AlphaTechini/doc-fetch/releases/download/v1.0.0/{binary_name}"
|
|
53
|
+
|
|
54
|
+
# Destination path
|
|
55
|
+
install_dir = os.path.join(self.install_scripts, 'doc-fetch-bin')
|
|
56
|
+
os.makedirs(install_dir, exist_ok=True)
|
|
57
|
+
|
|
58
|
+
if platform.system() == 'Windows':
|
|
59
|
+
binary_path = os.path.join(install_dir, 'doc-fetch.exe')
|
|
60
|
+
else:
|
|
61
|
+
binary_path = os.path.join(install_dir, 'doc-fetch')
|
|
62
|
+
|
|
63
|
+
print(f"Downloading DocFetch binary from {download_url}")
|
|
64
|
+
print(f"Platform: {platform.system()} {platform.machine()}")
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
# Download the binary
|
|
68
|
+
with urllib.request.urlopen(download_url) as response:
|
|
69
|
+
with open(binary_path, 'wb') as f:
|
|
70
|
+
f.write(response.read())
|
|
71
|
+
|
|
72
|
+
# Make executable on Unix-like systems
|
|
73
|
+
if platform.system() != 'Windows':
|
|
74
|
+
st = os.stat(binary_path)
|
|
75
|
+
os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
|
|
76
|
+
|
|
77
|
+
print(f"Successfully installed DocFetch binary to {binary_path}")
|
|
78
|
+
|
|
79
|
+
except Exception as e:
|
|
80
|
+
print(f"Failed to download binary: {e}")
|
|
81
|
+
print("Falling back to Go build from source...")
|
|
82
|
+
self.build_from_source()
|
|
83
|
+
|
|
84
|
+
def build_from_source(self):
|
|
85
|
+
"""Build from Go source if binary download fails."""
|
|
86
|
+
try:
|
|
87
|
+
# Check if Go is installed
|
|
88
|
+
import subprocess
|
|
89
|
+
subprocess.run(['go', 'version'], check=True, capture_output=True)
|
|
90
|
+
|
|
91
|
+
# Build from source
|
|
92
|
+
install_dir = os.path.join(self.install_scripts, 'doc-fetch-bin')
|
|
93
|
+
source_dir = os.path.join(os.path.dirname(__file__), '..')
|
|
94
|
+
|
|
95
|
+
if platform.system() == 'Windows':
|
|
96
|
+
binary_path = os.path.join(install_dir, 'doc-fetch.exe')
|
|
97
|
+
subprocess.run(['go', 'build', '-o', binary_path, './cmd/docfetch'],
|
|
98
|
+
cwd=source_dir, check=True)
|
|
99
|
+
else:
|
|
100
|
+
binary_path = os.path.join(install_dir, 'doc-fetch')
|
|
101
|
+
subprocess.run(['go', 'build', '-o', binary_path, './cmd/docfetch'],
|
|
102
|
+
cwd=source_dir, check=True)
|
|
103
|
+
|
|
104
|
+
# Make executable
|
|
105
|
+
if platform.system() != 'Windows':
|
|
106
|
+
st = os.stat(binary_path)
|
|
107
|
+
os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
|
|
108
|
+
|
|
109
|
+
print(f"Successfully built DocFetch from source: {binary_path}")
|
|
110
|
+
|
|
111
|
+
except Exception as e:
|
|
112
|
+
print(f"Failed to build from source: {e}")
|
|
113
|
+
print("Please install Go (https://golang.org/dl/) or ensure the binary is available for your platform.")
|
|
114
|
+
|
|
115
|
+
# Read README for long description
|
|
116
|
+
with open("README.md", "r", encoding="utf-8") as fh:
|
|
117
|
+
long_description = fh.read()
|
|
118
|
+
|
|
119
|
+
setup(
|
|
120
|
+
name="doc-fetch",
|
|
121
|
+
version="1.0.1",
|
|
122
|
+
author="AlphaTechini",
|
|
123
|
+
author_email="rehobothokoibu@gmail.com",
|
|
124
|
+
description="Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption",
|
|
125
|
+
long_description=long_description,
|
|
126
|
+
long_description_content_type="text/markdown",
|
|
127
|
+
url="https://github.com/AlphaTechini/doc-fetch",
|
|
128
|
+
packages=find_packages(),
|
|
129
|
+
classifiers=[
|
|
130
|
+
"Development Status :: 5 - Production/Stable",
|
|
131
|
+
"Intended Audience :: Developers",
|
|
132
|
+
"Intended Audience :: Information Technology",
|
|
133
|
+
"License :: OSI Approved :: MIT License",
|
|
134
|
+
"Operating System :: OS Independent",
|
|
135
|
+
"Programming Language :: Python :: 3",
|
|
136
|
+
"Programming Language :: Python :: 3.7",
|
|
137
|
+
"Programming Language :: Python :: 3.8",
|
|
138
|
+
"Programming Language :: Python :: 3.9",
|
|
139
|
+
"Programming Language :: Python :: 3.10",
|
|
140
|
+
"Programming Language :: Python :: 3.11",
|
|
141
|
+
"Topic :: Documentation",
|
|
142
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
143
|
+
"Topic :: Software Development :: Documentation",
|
|
144
|
+
"Topic :: Utilities",
|
|
145
|
+
],
|
|
146
|
+
python_requires=">=3.7",
|
|
147
|
+
install_requires=[],
|
|
148
|
+
entry_points={
|
|
149
|
+
"console_scripts": [
|
|
150
|
+
"doc-fetch=doc_fetch.cli:main",
|
|
151
|
+
],
|
|
152
|
+
},
|
|
153
|
+
cmdclass={
|
|
154
|
+
'install': InstallCommand,
|
|
155
|
+
},
|
|
156
|
+
include_package_data=True,
|
|
157
|
+
zip_safe=False,
|
|
158
|
+
)
|