j1-template 2022.2.2 → 2022.3.1

data/lib/starter_web/pages/public/jupyter/notebooks/binder/binder-launches.ipynb

@@ -1,385 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Analysing mybinder.org launches\n",
-    "\n",
-    "The first few cells download and massage the data. Later on we answer questions on which repositories are popular and such."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%matplotlib inline\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "import datetime\n",
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "index = pd.read_json(\"https://archive.analytics.mybinder.org/index.jsonl\",\n",
-    "                     lines=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# get all days since start of 2019\n",
-    "now = datetime.datetime.now()\n",
-    "n = (now - datetime.datetime(2019, 1, 1)).days\n",
-    "\n",
-    "# small range for previewing\n",
-    "#n = 30\n",
-    "\n",
-    "frames = []\n",
-    "for idx, day in index.sort_index(ascending=False).iterrows():\n",
-    "    df = pd.read_json(\"https://archive.analytics.mybinder.org/{}\".format(day['name']),\n",
-    "                      lines=True, )\n",
-    "    frames.append(df)\n",
-    "    if len(frames) > n:\n",
-    "        break\n",
-    "        \n",
-    "print(f\"Fetched data for {n} days.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = pd.concat(frames)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# make it easier to grab the ref\n",
-    "def get_repo(spec):\n",
-    "    s = spec.rsplit(\"/\", 1)[0]\n",
-    "    if s.endswith('.git'):\n",
-    "        s = s[:-4]\n",
-    "    return s\n",
-    "df['repo'] = df['spec'].apply(get_repo) #lambda s: s.rsplit(\"/\", 1)[0].replace(\".git\", \"\"))\n",
-    "df['org'] = df['spec'].apply(lambda s: s.split(\"/\", 1)[0])\n",
-    "df['ref'] = df['spec'].apply(lambda s: s.rsplit(\"/\", 1)[1])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# take a look at the data, does it look sensible?\n",
-    "df.sample(10)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = df.drop(columns=['schema', 'version', 'spec', 'status'])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df.head()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Monthly, weekly, daily active repos"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_ = df.set_index(\"timestamp\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def n_active_repos(period=30):\n",
-    "    \"\"\"Unique active repos over the period\n",
-    "    \n",
-    "    With period=30 this gives monthly active repos\n",
-    "    With period=1 you get daily active repos\n",
-    "    \"\"\"\n",
-    "    now = datetime.datetime.now()\n",
-    "    start = datetime.datetime(2019, 1, 1)\n",
-    "    days_since_start = (now - start).days\n",
-    "\n",
-    "    data = []\n",
-    "\n",
-    "    for n in range(days_since_start):\n",
-    "        s = start + datetime.timedelta(days=n)\n",
-    "        e = start + datetime.timedelta(days=n+period)\n",
-    "        if e > now:\n",
-    "            break\n",
-    "\n",
-    "        # our \"monthly\" data\n",
-    "        monthly = df_.loc[s.isoformat():e.isoformat()]\n",
-    "        data.append(dict(timestamp=e, repos=len(set(monthly.repo))))\n",
-    "        \n",
-    "    return pd.DataFrame.from_records(data).set_index(\"timestamp\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "monthly_active = n_active_repos(30)\n",
-    "weekly_active = n_active_repos(7)\n",
-    "daily_active = n_active_repos(1)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "activity = pd.merge(monthly_active, daily_active,\n",
-    "                    suffixes=(\"_monthly\", \"_daily\"), left_index=True, right_index=True)\n",
-    "activity = pd.merge(activity, weekly_active, left_index=True, right_index=True)\n",
-    "\n",
-    "activity.columns = ['30day active', '1d active', '7d active']\n",
-    "activity.plot();"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Total launches"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Sneak peek: total launches!\n",
-    "df.shape"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Launches per day"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "daily = df.set_index(\"timestamp\").resample('D').count()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "daily['repo'].plot()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Estimate number of unique repositories\n",
-    "\n",
-    "Expect the raw number of launches to be bigger than the number of repositories launched more than once. Those launched only once might have been accidents."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "len(set(df.repo))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from collections import Counter\n",
-    "\n",
-    "sum(1 for k,v in Counter(df.repo).items() if v > 3)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Popular repositories and their branches\n",
-    "\n",
-    "Twenty most popular repos:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "top20 = df.groupby(\"repo\").count().sort_values(\"timestamp\", ascending=False).head(20)\n",
-    "top20"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(\"Cumulative top20 launches:\", top20['ref'].sum())\n",
-    "print(\"The top20 repos are {:.1f}% of all \"\n",
-    "      \"launches.\".format(100 * top20['ref'].sum() / df.shape[0]))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Per org stats\n",
-    "\n",
-    "Needs more RAM than mybinder.org gives us for now"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# add a new column showing total launches per repo\n",
-    "totals_per_repo = (df.groupby([\"repo\"])\n",
-    " .size()\n",
-    " .reset_index(name='repo_counts'))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# add a nnew column showing total launches per org\n",
-    "totals_per_org = (df.groupby([\"org\"])\n",
-    " .size()\n",
-    " .reset_index(name='org_counts'))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_ = pd.merge(df, totals_per_repo, on='repo')\n",
-    "df_ = pd.merge(df_, totals_per_org, on='org')\n",
-    "#df_.sample(10)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "(df.groupby(\"provider\")\n",
-    "   .size()\n",
-    "   .reset_index(name='Launches')\n",
-    "   .sort_values('Launches', ascending=False))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "(df_.groupby([\"org\", \"repo\", \"ref\", \"repo_counts\", \"org_counts\"])\n",
-    " .size()\n",
-    " # give the column a nice name\n",
-    " .reset_index(name='ref_counts')\n",
-    " # sort first by total launches, then within a repo by ref launches\n",
-    " .sort_values(['org_counts', 'repo_counts', 'ref_counts'],\n",
-    "              ascending=[False,False, False])\n",
-    " .set_index([\"org\", 'repo', 'ref'])\n",
-    ")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

data/lib/starter_web/pages/public/jupyter/notebooks/j1/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -1,6 +0,0 @@
-{
- "cells": [],
- "metadata": {},
- "nbformat": 4,
- "nbformat_minor": 5
-}